Merge pull request 'Refuse a verb argument the seat would pass over; a push without a machine says it is the whole mesh (hq issue 244)' (#71) from fix/verb-schemas into main

A push naming one machine reached the verb without it and pushed every
machine behind (hq issue 244). The controller now refuses any argument a
verb does not declare, any it composed its command line without, and a
switch that is not true or false; a push that names no machine says first
that it is the whole mesh. Tests walk every served verb: no argument is
ever ignored, and every flag of a verb's command, read from the source, is
in its schema or accounted for. plan gains files, push behind, builds and
plans limit.

Dockerfile

@@ -1,5 +1,11 @@
-ARG GO_BASE=golang:1.25-alpine
+# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
-# The control plane's image.
+# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
+# `make image` broke once before (issue 146).
+ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
+# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
+# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
+# Genesis builds this file and raises it as the container the process replaces on the first push
+# (mesh-host internal/bootstrap, novox/hq issue 223).
 #
 # novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
 # machine where no mesh exists yet, and run before there is anything to check it against. So it

Makefile

@@ -27,17 +27,21 @@ build:
 IMAGE ?= mesh-controller:$(VERSION)
 DEV_TAG ?= mesh-controller:development
 
-# The base the module declares, read from the manifest rather than written here twice.
+# The Go base the image is built on.
 #
 # **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
 # older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
 # 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
 # building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
 # what it cost).
-GO_BASE ?= $(shell python3 -c "import json;print(next(o['image'] for o in json.load(open('module.json'))['build']['on'] if o['arg']=='GO_BASE'))" 2>/dev/null)
+#
+# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
+# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
+# still needs a Go base. The digest is the one the manifest declared until then.
+GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
 
 image:
-	@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
+	@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
 	docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
 	@echo
 	@docker image inspect $(IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
@@ -48,7 +52,7 @@ BUILDER_IMAGE ?= mesh-builder:$(VERSION)
 BUILDER_DEV_TAG ?= mesh-builder:development
 
 builder-image:
-	@test -n "$(GO_BASE)" || { echo "module.json declares no GO_BASE; pass GO_BASE=<image> or fix the manifest"; exit 1; }
+	@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
 	docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
 	@echo
 	@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'

README.md

@@ -193,6 +193,13 @@ passes every check that only looks at the message.
 
 ## The image
 
+**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
+bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
+`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
+still runs a container of the controller: genesis, which raises the first controller from it and
+installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
+mesh's own build any more — `make image` builds it.
+
 `FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
 manager, no libc, no CA certificates.
 

cmd/mesh-builder/holder.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go/micro"
+
+	"github.com/novox/mesh-controller/internal/builder"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
+//
+// **The queue is the controller's; the build running here is this machine's.** The controller can
+// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
+// on this machine and containers in this machine's runtime, and only this machine can end them. So
+// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
+// it, pause, resume.
+//
+// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
+// set per build — so "the build running here" is one or none, and kill names it by id so a call
+// that arrives as one build ends and the next begins cannot end the wrong one.
+
+// holder is this machine's state as a holder of the build seat.
+type holder struct {
+	on, seat string
+	// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
+	// rather than silently taking work again.
+	workspace string
+	// say publishes this machine's state: whether it takes work (link.HolderState).
+	say func(link.HolderState) error
+	// remove runs what a kill needs outside the build: the containers left behind.
+	remove builder.Runner
+
+	mu      sync.Mutex
+	paused  bool
+	running *running
+}
+
+// running is the build this machine is doing.
+type running struct {
+	request link.BuildRequest
+	step    string
+	started time.Time
+	cancel  context.CancelFunc
+	killed  bool
+	// returned is the build's own work having ended, before a kill or not; outcome is what was then
+	// announced, and announced whether it went out.
+	returned  bool
+	outcome   string
+	announced bool
+	done      chan struct{}
+}
+
+// pausedFile is where the flag lives in the workspace.
+func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
+
+// newHolder reads the paused flag the workspace keeps.
+func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
+	h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
+	if _, err := os.Stat(pausedFile(workspace)); err == nil {
+		h.paused = true
+	}
+	return h
+}
+
+// Paused is asked by the taking loop before every fetch.
+func (h *holder) Paused() bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.paused
+}
+
+// setPaused records the flag in the workspace first and then in memory, so what this holder says
+// it is and what it would be after a restart never differ.
+func (h *holder) setPaused(paused bool) error {
+	path := pausedFile(h.workspace)
+	if paused {
+		if err := os.MkdirAll(h.workspace, 0o755); err != nil {
+			return err
+		}
+		if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+			return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
+		}
+	} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
+	}
+	h.mu.Lock()
+	h.paused = paused
+	h.mu.Unlock()
+	h.announce()
+	return nil
+}
+
+// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
+// controller reading an older state is a plan read as late rather than a build lost.
+func (h *holder) announce() {
+	if h.say == nil {
+		return
+	}
+	if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
+		fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
+	}
+}
+
+// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
+// a pause outlives the events stream's retention.
+func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
+	h.announce()
+	tick := time.NewTicker(every)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			if h.Paused() {
+				h.announce()
+			}
+		}
+	}
+}
+
+// begin records the build this machine took, with the cancel that ends it.
+func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
+	r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
+	h.mu.Lock()
+	h.running = r
+	h.mu.Unlock()
+	return r
+}
+
+// end clears it, and lets a kill waiting on it know it is over.
+func (h *holder) end(r *running) {
+	h.mu.Lock()
+	if h.running == r {
+		h.running = nil
+	}
+	h.mu.Unlock()
+	close(r.done)
+}
+
+// stepped records the step a build is at, for `current`.
+func (h *holder) stepped(r *running, step string) {
+	h.mu.Lock()
+	r.step = step
+	h.mu.Unlock()
+}
+
+// currentBuild is what `current` answers.
+type currentBuild struct {
+	On      string `json:"on"`
+	Paused  bool   `json:"paused"`
+	Running *struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	} `json:"running,omitempty"`
+	Said string `json:"said"`
+}
+
+func (h *holder) current() currentBuild {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	out := currentBuild{On: h.on, Paused: h.paused}
+	taking := "taking builds"
+	if h.paused {
+		taking = "paused, taking no new build"
+	}
+	if h.running == nil {
+		out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
+		return out
+	}
+	r := h.running
+	elapsed := time.Since(r.started).Round(time.Second)
+	out.Running = &struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
+		r.started.UTC().Format(time.RFC3339), elapsed.String()}
+	out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
+		h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
+	return out
+}
+
+// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
+// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
+// (killAnswer in the controller's queue.go, 75s).
+var (
+	killRemoves = 15 * time.Second
+	killWaits   = 20 * time.Second
+)
+
+// kill ends the build with this id, if it is the one running here and still working: its context
+// cancelled — which kills each command's process group — and the containers it started removed by
+// their label, once at once and again after the build has ended, so one created while it was being
+// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
+// ask so it is not redelivered; the answer says whether that happened.
+func (h *holder) kill(id string) (string, error) {
+	h.mu.Lock()
+	r := h.running
+	if r == nil || r.request.ID != id {
+		doing := "nothing"
+		if r != nil {
+			doing = r.request.ID
+		}
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
+	}
+	if r.returned {
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
+	}
+	r.killed = true
+	cancel, done := r.cancel, r.done
+	h.mu.Unlock()
+
+	cancel()
+	removed, removeErr := h.removeContainers(id)
+	ended := false
+	select {
+	case <-done:
+		ended = true
+	case <-time.After(killWaits):
+	}
+	again, againErr := h.removeContainers(id)
+	removed += again
+	if removeErr == nil {
+		removeErr = againErr
+	}
+	containers := fmt.Sprintf("%d container(s) it started removed", removed)
+	if removeErr != nil {
+		containers = "its containers could not all be listed or removed: " + removeErr.Error()
+	}
+	if !ended {
+		return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
+			"its outcome is in", id, h.on, containers), nil
+	}
+	h.mu.Lock()
+	outcome, announced := r.outcome, r.announced
+	h.mu.Unlock()
+	if !announced {
+		return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
+			"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
+			containers), nil
+	}
+	return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
+		"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
+}
+
+// removeContainers is one pass of removing what the build left, bounded.
+func (h *holder) removeContainers(id string) (int, error) {
+	cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
+	defer stop()
+	return builder.RemoveContainersOf(cleanup, h.remove, id)
+}
+
+// returned records that the build's work ended, and says whether a kill came first — only then is
+// the build killed; an error it ended with on its own is its own outcome.
+func (h *holder) returned(r *running) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	r.returned = true
+	return r.killed
+}
+
+// said records the outcome announced, and whether it went out, for a kill to answer with.
+func (h *holder) said(r *running, outcome string, announced bool) {
+	h.mu.Lock()
+	r.outcome, r.announced = outcome, announced
+	h.mu.Unlock()
+}
+
+// handlers are the seat's verbs, as this machine answers them.
+func (h *holder) handlers() map[string]link.ToolHandler {
+	return map[string]link.ToolHandler{
+		"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
+		"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
+			var args struct {
+				ID string `json:"id"`
+			}
+			if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
+				return nil, errors.New("kill needs the build's id")
+			}
+			said, err := h.kill(strings.TrimSpace(args.ID))
+			if err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": said}, nil
+		},
+		"pause": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(true); err != nil {
+				return nil, err
+			}
+			said := h.on + " is paused: it takes no new build until resumed"
+			if c := h.current(); c.Running != nil {
+				said += "; " + c.Running.ID + " runs on and finishes"
+			}
+			return map[string]any{"said": said, "paused": true}, nil
+		},
+		"resume": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(false); err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
+		},
+	}
+}
+
+func orNothing(s string) string {
+	if s == "" {
+		return "its start"
+	}
+	return s
+}
+
+// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
+// log it would be is the one being ended.
+func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+	}
+	return string(out), nil
+}
+
+// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
+// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
+// the module answering, the seat, scope node, the machine, its description and argument schema — so
+// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
+//
+// One service per machine, named for the seat and identified by the machine, so the answer is this
+// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
+// store, and what it serves is what this binary was built to serve.
+func announcement(seat, module, node string) micro.Info {
+	s, _ := catalogue.SeatNamed(seat)
+	var endpoints []micro.EndpointInfo
+	for _, v := range s.Serves {
+		schema, _ := json.Marshal(v.Input)
+		endpoints = append(endpoints, micro.EndpointInfo{
+			Name:    seat + "__" + v.Name,
+			Subject: link.NodeSeatToolSubject(seat, v.Name, node),
+			// The queue group the verbs are served in, as every runtime announces its own.
+			QueueGroup: "seat." + seat,
+			Metadata: map[string]string{
+				"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
+				"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
+			},
+		})
+	}
+	return micro.Info{
+		ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
+			Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
+		Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
+		Endpoints:   endpoints,
+	}
+}
+
+// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
+func moduleOf(user string) string {
+	if _, module, ok := strings.Cut(user, "."); ok && module != "" {
+		return module
+	}
+	return "build-agent"
+}

cmd/mesh-builder/holder_test.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
+// 0219), and what it says about itself follows.
+func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
+	workspace := t.TempDir()
+	var said []link.HolderState
+	h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
+		said = append(said, s)
+		return nil
+	})
+	if h.Paused() {
+		t.Fatal("a new holder starts paused")
+	}
+	if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
+		t.Fatalf("paused: %v, said %+v", h.Paused(), said)
+	}
+	again := newHolder("ace", link.TheBuildMachine, workspace, nil)
+	if !again.Paused() {
+		t.Fatal("restarted, the holder forgot it was paused")
+	}
+	if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
+		t.Fatal("resumed, the holder came back paused")
+	}
+}
+
+// kill ends the build with that id — its context, which ends its commands — removes what it left by
+// label, and refuses an id it is not building.
+func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	var removed []string
+	h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		removed = append(removed, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\n", nil
+		}
+		return "", nil
+	}
+	kill := h.handlers()["kill"]
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
+		t.Fatal("killed a build while none ran")
+	}
+
+	building, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
+	h.stepped(r, "image")
+	if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
+		t.Fatalf("current says %+v", c)
+	}
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
+		!strings.Contains(err.Error(), "build-1") {
+		t.Fatalf("killed another id: %v", err)
+	}
+	// The build's own goroutine: it ends when its context does, as a build's commands do, and
+	// announces what came of it.
+	go func() {
+		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, true)
+		}
+		h.end(r)
+	}()
+	answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if building.Err() == nil {
+		t.Fatal("the build's context was not cancelled")
+	}
+	if !r.killed {
+		t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
+	}
+	said := answer.(map[string]any)["said"].(string)
+	if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
+		t.Errorf("kill said %q", said)
+	}
+	// Removed at the kill and again once the build had ended: a container made in between is caught.
+	if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
+		t.Errorf("removed %v", removed)
+	}
+	if c := h.current(); c.Running != nil {
+		t.Errorf("after the kill current says %+v", c)
+	}
+}
+
+// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
+func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
+	info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
+	if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
+		t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
+	}
+	kill := info.Endpoints[1]
+	md := kill.Metadata
+	if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
+		md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
+		!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
+		t.Fatalf("kill is announced as %+v", kill)
+	}
+	body, err := json.Marshal(info)
+	if err != nil || len(body) > 8*1024 {
+		t.Fatalf("the answer is %d bytes (%v)", len(body), err)
+	}
+}
+
+// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
+// own error is its outcome. One whose outcome could not be announced is not said to be settled.
+func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
+	_, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
+	if killed := h.returned(r); killed {
+		t.Fatal("a build nobody killed reads as killed")
+	}
+	if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
+		t.Fatalf("killed a build that had ended: %v", err)
+	}
+	h.end(r)
+
+	building, cancel := context.WithCancel(context.Background())
+	r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
+	go func() {
+		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, false)
+		}
+		h.end(r)
+	}()
+	said, err := h.kill("build-2")
+	if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
+		t.Fatalf("kill said %q (%v)", said, err)
+	}
+}

cmd/mesh-builder/main.go

@@ -19,11 +19,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/builder"
@@ -107,48 +109,96 @@ func run() error {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	machine, err := takeWorkFrom(credential, on)
+	js, seat, err := dialFor(credential)
 	if err != nil {
 		return err
 	}
+	defer js.Close()
+
+	// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
+	// paused flag is read from the workspace before anything is taken, so a holder restarted while
+	// paused takes nothing.
+	h := newHolder(on, seat, workspace, func(state link.HolderState) error {
+		body, err := json.Marshal(state)
+		if err != nil {
+			return err
+		}
+		_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
+		return err
+	})
+	if h.Paused() {
+		fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
+	}
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
 	defer machine.Close()
 
+	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
+	// one serves none, and a subscription its holder has no grant for would be refused for ever.
+	if seat == link.TheBuildMachine {
+		stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
+			log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopServing()
+		// And says so, for the console to find (novox/hq ADR 0197).
+		stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
+			announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopAnnouncing()
+		go h.stateWhilePaused(ctx, time.Hour)
+	}
+
 	fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
-		answer(ctx, publisher, on, workspace, work)
+		answer(ctx, publisher, on, workspace, work, h)
 	})
 }
 
-// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
+// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
+// holds.
 //
 // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
 // machine told about both would take work from one and answer on the other, and every log line would
 // say it was fine.
-func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
+func dialFor(credential Credential) (*broker.JetStream, string, error) {
 	// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
 	// A credential for the mesh's bus carries user, password and fingerprint beside the address,
 	// and that is enough to dial it, pinned.
 	if !credential.onTheNewBus() {
-		return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
+		return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
 	}
 	js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
 	// handover): the mesh issues a build machine's credential naming the seat its module claims,
 	// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
 	seat := link.BuildSeatClaimed(credential.seatsClaimed())
 	fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
-	return link.MachineOverNATSOn(js, on, seat), nil
+	return js, seat, nil
 }
 
 // answer does one build and says what happened, whichever way it went.
-func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
+func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
 	request := work.Request()
 
+	// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
+	// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
+	// reaches it through the parent, and that keeps today's meaning below.
+	building, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var mine *running
+	if h != nil {
+		mine = h.begin(request, cancel)
+		defer h.end(mine)
+	}
+
 	// **First thing, and to stdout.** A build request that arrives and produces no visible line until
 	// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
 	// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
@@ -162,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	say := func(step, message string) {
 		fmt.Fprintf(os.Stderr, "  [%s] %s\n", step, message)
 		work.Say(step, message)
+		if mine != nil && step != "run" && step != "output" {
+			h.stepped(mine, step)
+		}
 	}
 	builder.Said = say
 	defer func() { builder.Said = nil }()
@@ -171,7 +224,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 
 	result := link.BuildResult{
 		ID: request.ID, Repository: request.Repository, Path: request.Path,
-		Ref: request.Ref, On: on, Source: request.Source,
+		Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
 	}
 	what := "building " + request.Repository
 	if request.Path != "" {
@@ -188,11 +241,24 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// The package-registry credential is a build input, so it is resolved before the clone: a
 		// build that could not have resolved its dependencies is refused in front of the reason, not
 		// after a clone that then fails at npm ci.
-		built, err = builder.Build(ctx, builder.Command, publisher,
+		// Every container it starts is labelled with its id, so a kill finds what outlived the
+		// docker client (ADR 0219).
+		built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
 			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
 			forgeFrom(), say, request.Seats)
 	}
-	if err != nil {
+	// Only a build the kill ended: the kill came before its work did. One that finished — built, or
+	// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
+	killed := false
+	if mine != nil {
+		killed = h.returned(mine) && err != nil
+	}
+	if killed {
+		// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
+		// error it ended with is the kill's consequence, not a fault of the source.
+		result.Failed = link.KilledByHand
+		say("failed", link.KilledByHand)
+	} else if err != nil {
 		// A failure is a result. A build that fails and says nothing is indistinguishable from a
 		// builder that is not running, and those want completely different responses.
 		result.Failed = err.Error()
@@ -217,7 +283,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		}
 	}
 
-	if err := work.Announce(ctx, result); err != nil {
+	// A killed build is announced on a context of its own: the build's was the one cancelled, and the
+	// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
+	// to be built again. A machine being stopped is the other case and keeps its meaning: the
+	// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
+	announcing := ctx
+	if killed {
+		fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
+		defer stop()
+		announcing = fresh
+	}
+	announceErr := work.Announce(announcing, result)
+	if mine != nil {
+		h.said(mine, result.Failed, announceErr == nil)
+	}
+	if err := announceErr; err != nil {
 		// Said, not fatal: the build happened. A build reported as failed because announcing it
 		// failed is a lie about work that was done — and the request stays unsettled below only if
 		// nothing was said at all, so another machine can try.

cmd/mesh-controller/acts.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/novox/mesh-controller/internal/broker"
+	"slices"
 	"sort"
 	"strings"
 
@@ -39,7 +40,10 @@ import (
 //
 // It costs a resolution per machine. Assignment is a person typing a command, and being told which
 // machines this just blocked is worth more than the milliseconds.
-func assign(ctx context.Context, open *stores, node, module string) (string, error) {
+func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
+	if len(modules) == 0 {
+		return "", fmt.Errorf("assign %s names no module", node)
+	}
 	// Held while it is recorded, so it cannot land between a converge's preview and its flip and
 	// be taken without ever having been previewed (novox/hq ADR 0100).
 	ctx, release, err := holdNodes(ctx, open, []string{node})
@@ -47,6 +51,16 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
 		return "", err
 	}
 	defer release()
+	// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
+	// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
+	// resources are applied through a seat nothing on the node holds is refused, because that order
+	// — the service manager, the package manager and the runtime before anything that installs,
+	// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
+	// holders that depend on each other go on in one command.
+	shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
+	if err != nil {
+		return "", err
+	}
 	// **Before the new assignment can unsettle a seat somebody holds only by being alone**
 	// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
 	// assignment resolves against a record rather than against a coincidence.
@@ -54,65 +68,176 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
 	if err != nil {
 		return "", err
 	}
-	fresh, err := open.inventory.Assign(ctx, node, module)
+	var lines []string
-	if err != nil {
+	var added []string
-		return "", err
+	for _, module := range modules {
+		fresh, err := open.inventory.Assign(ctx, node, module)
+		if err != nil {
+			return strings.Join(lines, "\n"), err
+		}
+		if !fresh {
+			// Nothing changed, and saying "is assigned" would read as an action. One node runs one
+			// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
+			lines = append(lines, fmt.Sprintf(
+				"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
+			continue
+		}
+		added = append(added, module)
+		lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
 	}
-	if !fresh {
+	if len(added) == 0 {
-		// Nothing changed, and saying "is assigned" would read as an action. One node runs one
+		return strings.Join(lines, "\n"), nil
-		// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
-		return fmt.Sprintf("%s already runs %s — one node runs one of each (ADR 0115); nothing changed",
-			node, module), nil
 	}
-	said := fmt.Sprintf("%s is assigned %s", node, module)
+	answer := strings.Join(lines, "\n")
 	for _, line := range settled {
-		said += "\n  " + line
+		answer += "\n  " + line
+	}
+	// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
+	// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
+	// node's list, and every other node's, is `status`'s.
+	for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
+		answer += "\n  " + line
 	}
 	// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
 	// credential exists delivers a process that cannot authenticate and crash-loops until somebody
 	// runs a second verb and a second push. Issued here when the module speaks on the bus and has
 	// no credential yet; kept when it has one, so re-assigning rotates nothing.
-	if line := issueOnAssign(ctx, open, node, module); line != "" {
+	for _, module := range added {
-		said += "\n  " + line
+		if line := issueOnAssign(ctx, open, node, module); line != "" {
+			answer += "\n  " + line
+		}
 	}
 	plan, _, err := planFor(ctx, open, node)
 	if err != nil {
 		// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
 		// worth reporting: this machine's refusal is rarely the only consequence.
-		return said + blockedElsewhere(ctx, open, node), err
+		return answer + blockedElsewhere(ctx, open, node), err
 	}
 	// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
 	// capability the machine lacks is on the wrong machine, and the assignment records what a person
 	// meant while this line says it will not run until it moves. The rest of the node still pushes.
+	isAdded := map[string]bool{}
+	for _, m := range added {
+		isAdded[m] = true
+	}
 	for _, u := range plan.Unhostable {
-		if u.Module != module {
+		if !isAdded[u.Module] {
 			continue
 		}
 		for _, c := range u.Missing {
-			said += "\n  but " + catalogue.WrongMachine(u.Module, c, node)
+			answer += "\n  but " + catalogue.WrongMachine(u.Module, c, node)
 		}
 	}
-	return said + fmt.Sprintf("\n  run `push %s` to send it", node) +
+	return answer + fmt.Sprintf("\n  run `push %s` to send it", node) +
 		blockedElsewhere(ctx, open, node), nil
 }
 
-// unassign takes a module off a node. What it leaves behind is the host's business: a directory
+// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
+// catalogue and the node's assignments it was judged against. Modules already assigned are not new
+// and are not judged again.
+func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
+	map[string]catalogue.Manifest, []string, error) {
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	assigned, err := open.inventory.Assigned(ctx, node)
+	if err != nil {
+		return nil, nil, err
+	}
+	already := map[string]bool{}
+	for _, a := range assigned {
+		already[a] = true
+	}
+	var adding []string
+	for _, m := range modules {
+		if !already[m] {
+			adding = append(adding, m)
+		}
+	}
+	// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
+	// with everything else this act changed, so they are not said twice.
+	if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
+		return nil, nil, err
+	}
+	// Two modules declaring one package, path or unit is refused before anything is recorded
+	// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
+	if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
+		return nil, nil, err
+	}
+	return shelf, assigned, nil
+}
+
+// unassign takes modules off a node. What they leave behind is the host's business: a directory
 // holding anything the mesh did not put there is kept (novox/hq ADR 0030).
 //
 // It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
 // module off one machine is the ordinary way to stop providing something to another, and nothing
 // about the command's own output would ever have said so.
-func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
+//
+// **Refused when it takes away the last holder of a seat a module left on the node depends on**
+// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
+// modules in one act are judged together, so a holder and its dependents come off in one command.
+func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
+	if len(modules) == 0 {
+		return "", fmt.Errorf("unassign %s names no module", node)
+	}
 	ctx, release, err := holdNodes(ctx, open, []string{node})
 	if err != nil {
 		return "", err
 	}
 	defer release()
-	if err := open.inventory.Unassign(ctx, node, module); err != nil {
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
 		return "", err
 	}
-	return fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
+	assigned, err := open.inventory.Assigned(ctx, node)
-		node, module, node) + blockedElsewhere(ctx, open, node), nil
+	if err != nil {
+		return "", err
+	}
+	// Every one checked before any is taken off, so a refusal leaves the node as it was.
+	runs := map[string]bool{}
+	for _, a := range assigned {
+		runs[a] = true
+	}
+	for _, module := range modules {
+		if !runs[module] {
+			return "", fmt.Errorf("%s is not assigned to %s", module, node)
+		}
+	}
+	if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
+		return "", err
+	}
+	for _, module := range modules {
+		if err := open.inventory.Unassign(ctx, node, module); err != nil {
+			return "", err
+		}
+	}
+	answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
+		node, strings.Join(modules, ", "), node)
+	var left []string
+	for _, a := range assigned {
+		if !slices.Contains(modules, a) {
+			left = append(left, a)
+		}
+	}
+	for _, line := range unheldChange(shelf, node, assigned, left) {
+		answer += "\n  " + line
+	}
+	return answer + blockedElsewhere(ctx, open, node), nil
+}
+
+// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
+// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
+// command API and the controller seat's verbs as they do from the command line.
+func splitModules(field string) []string {
+	var out []string
+	for _, m := range strings.Split(field, ",") {
+		if m = strings.TrimSpace(m); m != "" {
+			out = append(out, m)
+		}
+	}
+	return out
 }
 
 // blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.

cmd/mesh-controller/addresses_test.go

@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
 	if _, err := assign(ctx, open, "laptop", "app"); err != nil {
 		t.Fatal(err)
 	}
+	// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
+	// asks where this machine reaches the store, as the docker module does.
+	register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
+		Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
+			"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
+	if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
+		t.Fatal(err)
+	}
 
 	on := map[string]bool{"anchor": true, "laptop": true}
 	node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
@@ -145,7 +153,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
 //
 // Composed from the control plane's own manifest against a real inventory: the store's module is
 // given 6852 on this node the way genesis or an operator gives it, and the control plane's
-// container is told so beside the sealed connection genesis wrote.
+// process is told so beside the sealed connection genesis wrote.
 func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
@@ -157,8 +165,8 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
 	if err != nil {
 		t.Fatal(err)
 	}
-	control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
+	control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
-		Reference: "registry.example/control@" + aDigest}})
+		Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,12 +208,12 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
 
 	var env map[string]any
 	for _, r := range composed(t, open, "anchor").Resources {
-		if r["id"] == "mesh-controller.server" {
+		if r["id"] == "mesh-controller.controller" {
 			env, _ = r["env"].(map[string]any)
 		}
 	}
 	if env == nil {
-		t.Fatal("the control plane's container is not in its own node's declaration")
+		t.Fatal("the control plane's process is not in its own node's declaration")
 	}
 	for key, want := range map[string]string{
 		"MESH_STORE_INVENTORY_PORT": "6852",
@@ -238,7 +246,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
 	out := m
 	out.Resources = nil
 	for _, r := range m.Resources {
-		if r["type"] != "container" {
+		if r["type"] != "container" && r["type"] != "process" {
 			out.Resources = append(out.Resources, r)
 			continue
 		}

cmd/mesh-controller/adopting_test.go

@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
+	if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{

cmd/mesh-controller/api.go

@@ -95,10 +95,10 @@ func commands(who Authenticator) http.Handler {
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
-		return assign(ctx, open, in.Node, in.Module)
+		return assign(ctx, open, in.Node, splitModules(in.Module)...)
 	}))
 	mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
-		return unassign(ctx, open, in.Node, in.Module)
+		return unassign(ctx, open, in.Node, splitModules(in.Module)...)
 	}))
 	// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
 	mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {

cmd/mesh-controller/build.go

@@ -392,22 +392,34 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
 //
 // Separated from the command so `--behind` can walk a list without a second path to the same act.
 func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
+	_, err := buildOneAsked(ctx, source, path, ref, wait, false)
+	return err
+}
+
+// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
+// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
+// at, never taken in (issue 240), which is what `replay` asks unless told to register.
+func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
+	dryRun bool) (string, error) {
+	if dryRun && wait != 0 {
+		return "", errors.New("a dry run waited for is `build --dry-run`")
+	}
 	// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
 	// the reason, rather than sent to a machine to fail at `git clone`.
 	repository, err := cloneFrom(ctx, source)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	ident, err := openIdentity(ctx)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer ident.Close()
 
 	server, err := connectLink(ctx, nil, nil, nil)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer server.Close()
 
@@ -420,6 +432,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		Ref:        ref,
 		Held:       heldBy(ctx),
 		Seats:      seatBases(ctx),
+		DryRun:     dryRun,
 	}
 	fmt.Printf("asked for %s", source)
 	if source.Seat != "" {
@@ -438,7 +451,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 	seat := buildSeatHeld(ctx)
 	ask, err := askOverOn(seat)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer ask.Close()
 	fmt.Printf("  of %s\n", seat)
@@ -449,26 +462,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		// is still here. A tool call cannot hold a connection for the minutes a build takes; it
 		// follows the build by its id instead.
 		if err := ask.Ask(ctx, request); err != nil {
-			return err
+			return "", err
+		}
+		if dryRun {
+			fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
+				"its outcome is not taken in\n", request.ID)
+			return request.ID, nil
 		}
 		fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
 			"shows what came of it; the module is registered when the outcome comes\n", request.ID)
-		return nil
+		return request.ID, nil
 	}
 
 	result, err := ask.Submit(ctx, request, wait)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 
 	open, err := openStores(ctx)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 	defer open.Close()
 	manifest, kept, err := takeIn(ctx, open.inventory, result)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 	// Said as recorded: what each artifact is, not where this builder happened to push it.
 	for _, made := range kept.Made {
@@ -478,7 +496,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		manifest.Module, manifest.Version, result.On, short(result.Commit))
 	saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
 	fmt.Printf("  run `assign <node> %s` to put it somewhere\n", manifest.Module)
-	return nil
+	return request.ID, nil
 }
 
 // saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
@@ -558,6 +576,10 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
 		}
 		return manifest, kept, err
 	}
+	// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
+	// than on a timer of its own: this is the only moment either is true. Never fatal — the build
+	// worked and the module is registered.
+	collect(ctx, inv)
 	return manifest, kept, nil
 }
 
@@ -588,6 +610,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
 		ID:         link.NewBuildID(time.Now()),
 		Repository: repository, Path: path, Ref: ref,
 		Held: heldBy(ctx), Seats: seatBases(ctx),
+		DryRun: true,
 	}, wait)
 	if err != nil {
 		return err
@@ -623,6 +646,8 @@ type answers struct {
 	reported []inventory.Reported
 	// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
 	plans []inventory.Plan
+	// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
+	paused pauseView
 	// refused is why a machine cannot be worked out at all, by name. A different thing from every
 	// other answer here: those are about a machine that was told something, and this is about one
 	// that cannot be told anything — it never reaches waiting, because nothing was computed for it
@@ -646,6 +671,14 @@ type answers struct {
 	// public name on the machine went dark. The holds were correct; they were recorded only in the
 	// machine's own state file, and the one visible symptom was a count that did not add up.
 	untaken map[string]map[string]int
+	// unheld is every module on a machine whose resources are applied through a seat nothing on
+	// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
+	// refused, until the switch — and while there is any, the mesh is not all well: the order the
+	// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
+	unheld []catalogue.Unheld
+	// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
+	// journal was the only place that said so for a day (04-ISSUES/179).
+	failing []inventory.ProviderStanding
 }
 
 // heldBy is every artifact this mesh has built, for a build that may need one as its base.

cmd/mesh-controller/check.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
 )
@@ -90,6 +91,17 @@ func moduleCheck(paths []string, out io.Writer) error {
 		if len(m.Invokes) > 0 {
 			fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
 		}
+		// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
+		if len(m.State) > 0 {
+			kept := make([]string, 0, len(m.State))
+			for _, s := range m.State {
+				kept = append(kept, s.Name)
+			}
+			fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
+		}
+		if len(m.Reads) > 0 {
+			fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
+		}
 		fmt.Fprintln(out)
 	}
 	if failed > 0 {

cmd/mesh-controller/collect.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
+//
+// **Run where the records change.** A build is the moment new bytes landed in the store and the
+// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
+// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
+//
+// Never fatal to a build. The build succeeded, the module is registered, and a store that could
+// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
+// again, and the references it could not collect are still uncollected, so nothing is lost by
+// having failed.
+
+// collect asks the store to let go of everything the mesh made and no longer keeps, and records
+// what it let go of. Says what it did and what it could not; returns nothing, because nothing
+// upstream should branch on it.
+func collect(ctx context.Context, inv *inventory.Inventory) {
+	references, err := inv.ToCollect(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
+		return
+	}
+	kept, err := inv.KeptArchives(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
+		return
+	}
+	if len(references) == 0 && len(kept) == 0 {
+		return
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
+		return
+	}
+	// As the mesh reaches it from the network. Empty means the store is not on the network — on a
+	// mesh being raised it is not yet, and there the store holds one build of anything and has
+	// nothing to collect.
+	address, err := artifactStoreAddress(ctx, inv, shelf, "")
+	if err != nil || address == "" {
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
+		}
+		return
+	}
+
+	// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
+	// never collected has the whole history to get through, and a person waiting on `build` should
+	// not pay for it. Two bounds, and what is left over is simply offered again next time —
+	// builds are frequent, and the point is that the store stops growing, not that it empties
+	// tonight.
+	within, stop := context.WithTimeout(ctx, sweepBudget)
+	defer stop()
+	store := artifacts.Store{Address: address}
+
+	// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
+	// manifest names, and archives were published as bare blobs, so every kept archive is first
+	// held by its manifest — which backfills the ones published before holders, a few at a time
+	// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
+	// the sweep before it deletes anything: "everything kept is held" is the precondition the
+	// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
+	wrote, missing, err := holdKept(within, store, kept)
+	if wrote > 0 {
+		fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
+	}
+	if missing > 0 {
+		fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
+			"`collection` lists them\n", missing)
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
+		return
+	}
+
+	var done []string
+	var left, skipped int
+	for i, reference := range references {
+		if i >= mostPerSweep || within.Err() != nil {
+			left = len(references) - i
+			break
+		}
+		err := store.LetGo(within, reference)
+		if err == nil || errors.Is(err, artifacts.Gone) {
+			// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
+			// again for ever.
+			done = append(done, reference)
+			continue
+		}
+		if errors.Is(err, artifacts.ErrNotOurs) {
+			// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
+			// marked collected — the mesh did not remove it and should not claim to — and not a
+			// reason to stop, because the store was never asked. One of these at the front of
+			// the oldest-first order ended every sweep until this.
+			skipped++
+			if skipped == 1 {
+				fmt.Fprintf(os.Stderr,
+					"the sweep will not address %s and went on: %v\n", reference, err)
+			}
+			continue
+		}
+		// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
+		// one refuses all of them — deletion disabled, the store down, the network gone — so
+		// going on would be a hundred identical failures and a hundred identical log lines in
+		// front of whoever was building something.
+		fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
+			reference, err)
+		left = len(references) - i
+		break
+	}
+
+	if len(done) > 0 {
+		// Recorded outside `within`: the deletions happened, and losing the record of them because
+		// the sweep ran out of budget would mean asking about them again for ever.
+		if err := inv.MarkCollected(ctx, done); err != nil {
+			fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
+				len(done), err)
+			return
+		}
+		fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
+			len(done))
+	}
+	if left > 0 {
+		fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
+	}
+	if skipped > 0 {
+		fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
+	}
+}
+
+// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
+// Answers how many holders it wrote and how many kept archives the store does not have.
+//
+// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
+// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
+// reference the store cannot be asked about is skipped as the deletion loop skips one
+// (novox/hq issue 226).
+func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
+	for _, reference := range kept {
+		if err := ctx.Err(); err != nil {
+			return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
+		}
+		did, err := store.Hold(ctx, reference)
+		switch {
+		case err == nil:
+			if did {
+				wrote++
+			}
+		case errors.Is(err, artifacts.Gone):
+			missing++
+		case errors.Is(err, artifacts.ErrNotOurs):
+		default:
+			return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
+		}
+	}
+	return wrote, missing, nil
+}
+
+// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
+// several times a day converges within days of this landing; small enough that no single build
+// waits on the whole backlog.
+const mostPerSweep = 200
+
+// sweepBudget is the longest a sweep will keep a build waiting.
+const sweepBudget = 60 * time.Second

cmd/mesh-controller/collection.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+)
+
+// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
+// (novox/hq issue 253, ADR 0189).
+//
+// **The question to answer before the store's collector runs for real.** The collector deletes
+// every blob no manifest names, and archives were published as bare blobs, so the nightly step
+// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
+// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
+// dry run go.
+//
+// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
+// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
+// new verb in the seat's row.
+
+type collectionReport struct {
+	// Store is the artifact store as this machine reached it; empty when it is not on the network.
+	Store string `json:"store"`
+	// KeptArchives is how many archives the mesh keeps, for either reason.
+	KeptArchives int `json:"kept_archives"`
+	// Held is how many of them the store holds by their manifest.
+	Held int `json:"held"`
+	// Unheld are the kept archives the collector would delete tonight if it ran for real.
+	Unheld []string `json:"unheld"`
+	// Missing are kept archives the store does not have at all.
+	Missing []string `json:"missing"`
+	// Unasked is how many could not be asked about, and why the asking stopped.
+	Unasked int    `json:"unasked"`
+	Stopped string `json:"stopped,omitempty"`
+	// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
+	// collected — split by kind.
+	Eligible         int `json:"eligible"`
+	EligibleImages   int `json:"eligible_images"`
+	EligibleArchives int `json:"eligible_archives"`
+	// SafeToCollect is whether every kept archive was asked about and every one is held.
+	SafeToCollect bool `json:"safe_to_collect"`
+}
+
+func collectionCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("collection", flag.ContinueOnError)
+	asJSON := set.Bool("json", false, "answer as JSON")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) != 0 {
+		return errors.New("collection [--json]")
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	kept, err := inv.KeptArchives(ctx)
+	if err != nil {
+		return err
+	}
+	eligible, err := inv.ToCollect(ctx)
+	if err != nil {
+		return err
+	}
+	report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
+	for _, reference := range eligible {
+		if strings.Contains(reference, "/blobs/") {
+			report.EligibleArchives++
+		} else {
+			report.EligibleImages++
+		}
+	}
+
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return err
+	}
+	report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
+	if err != nil {
+		return err
+	}
+	if report.Store == "" {
+		report.Unasked = len(kept)
+		report.Stopped = "this mesh has no artifact store on its network"
+	} else {
+		report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
+	}
+	report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
+
+	if *asJSON {
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		return encoder.Encode(report)
+	}
+	printCollection(report)
+	return nil
+}
+
+// askHeld asks the store about each kept archive, stopping at the first answer that is not about
+// the archive: a store that cannot be reached for one cannot be for the next, and a page of
+// identical failures says less than one line.
+func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
+	for i, reference := range kept {
+		held, err := store.Held(ctx, reference)
+		switch {
+		case err == nil && held:
+			report.Held++
+		case err == nil:
+			report.Unheld = append(report.Unheld, reference)
+		case errors.Is(err, artifacts.Gone):
+			report.Missing = append(report.Missing, reference)
+		case errors.Is(err, artifacts.ErrNotOurs):
+			// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
+			report.Unasked++
+		default:
+			return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
+		}
+	}
+	return report.Unasked, ""
+}
+
+func printCollection(r collectionReport) {
+	store := r.Store
+	if store == "" {
+		store = "(not on the network)"
+	}
+	fmt.Printf("artifact store       %s\n", store)
+	fmt.Printf("kept archives        %d\n", r.KeptArchives)
+	fmt.Printf("  held               %d\n", r.Held)
+	fmt.Printf("  unheld             %d\n", len(r.Unheld))
+	fmt.Printf("  missing            %d\n", len(r.Missing))
+	if r.Unasked > 0 {
+		fmt.Printf("  not asked          %d (%s)\n", r.Unasked, r.Stopped)
+	}
+	fmt.Printf("eligible to let go   %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
+	if len(r.Unheld) > 0 {
+		fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
+		for _, reference := range r.Unheld {
+			fmt.Printf("  %s\n", reference)
+		}
+	}
+	if len(r.Missing) > 0 {
+		fmt.Println("\nmissing — kept by the mesh, not in the store:")
+		for _, reference := range r.Missing {
+			fmt.Printf("  %s\n", reference)
+		}
+	}
+	fmt.Println()
+	if r.SafeToCollect {
+		fmt.Println("every kept archive is held: the store's collector may run for real")
+	} else {
+		fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
+	}
+}

cmd/mesh-controller/delivery_order_test.go

@@ -0,0 +1,145 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
+type recordingDelivery struct {
+	did        []string
+	grantErr   error
+	declareErr error
+}
+
+func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
+	for _, s := range sending {
+		r.did = append(r.did, "grant "+s.node)
+	}
+	return r.grantErr
+}
+
+func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
+	if r.declareErr != nil {
+		return "", r.declareErr
+	}
+	r.did = append(r.did, "declare "+s.node)
+	return "digest-" + s.node, nil
+}
+
+func ready(names ...string) []readyNode {
+	var out []readyNode
+	for _, n := range names {
+		out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
+	}
+	return out
+}
+
+// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
+// machine is sent the code that uses them — except the machine holding the bus, whose declaration
+// carries the controller's own right to issue them, and goes first.
+func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
+	d := &recordingDelivery{}
+	digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
+	if !reflect.DeepEqual(d.did, want) {
+		t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
+	}
+	if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
+		t.Fatalf("the digests sent were not answered: %v", digests)
+	}
+
+	held := &recordingDelivery{}
+	if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
+		t.Fatal(err)
+	}
+	want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
+	if !reflect.DeepEqual(held.did, want) {
+		t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
+	}
+}
+
+// A grant that cannot be issued holds back the machines it concerns and is an error the caller
+// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
+// grant that would let the controller issue memberships is never held behind them.
+func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
+	// A failure naming no machine (the buckets): everything but the bus's machine.
+	d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
+	_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
+	if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
+		!strings.Contains(err.Error(), "anchor, laptop not sent") {
+		t.Fatalf("a failed grant was not said as the send's failure: %v", err)
+	}
+	if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
+		t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
+	}
+
+	// A membership that failed for one machine: that machine alone.
+	one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
+	_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
+	if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
+		t.Fatalf("one machine's refused membership was not said: %v", err)
+	}
+	if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
+		t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
+	}
+
+	// The announced upgrade that hit it is asked again.
+	if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
+		t.Fatal("an announcement whose send stopped at its grants is not asked again")
+	}
+	if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
+		t.Fatal("any failure is asked again, not only a grant's")
+	}
+
+	// Nothing to send is nothing granted either.
+	none := &recordingDelivery{grantErr: errors.New("never asked")}
+	if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
+		t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
+	}
+}
+
+// Whether the bus's machine goes first is read from the user list alone, by its digest.
+func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
+	list := "users: [a, b]"
+	if userListBehind(list, digestOf([]byte(list))) {
+		t.Fatal("the list it was sent reads as behind")
+	}
+	if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
+		t.Fatal("a changed or never-sent list reads as current")
+	}
+	if userListBehind("", "") {
+		t.Fatal("a machine sent no list reads as behind")
+	}
+}
+
+// The machine holding the bus goes first: its declaration carries the user list the new grants are
+// checked against. Among the machines it is moved to the front; not among them it is added only
+// when it is behind.
+func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
+	for _, c := range []struct {
+		what   string
+		names  []string
+		holder string
+		behind bool
+		want   []string
+	}{
+		{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
+		{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
+		{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
+		{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
+		{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
+	} {
+		if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
+		}
+	}
+}

cmd/mesh-controller/dry_run_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
+// store at all, so anything that tried to record or register would fail rather than pass quietly.
+func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
+	err := builds{}.Built(t.Context(), link.BuildResult{
+		ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
+		Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
+	})
+	if err != nil {
+		t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
+	}
+}
+
+// The mark survives the wire both ways: asked as a dry run, answered as one.
+func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
+	raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
+	var asked link.BuildRequest
+	if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
+		t.Fatalf("the request lost its dry-run mark: %s", raw)
+	}
+	raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
+	var answered link.BuildResult
+	if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
+		t.Fatalf("the outcome lost its dry-run mark: %s", raw)
+	}
+	raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
+	if string(raw) != `{"id":"build-2","repository":"","on":""}` {
+		t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
+	}
+}

cmd/mesh-controller/held_back.go

@@ -0,0 +1,241 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
+// issue 259, ADR 0221).
+//
+// A named push ends by sending every other machine whose declaration differs from what it was last
+// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
+// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
+// out makes every machine running it differ from the merge on, and so did a module whose plan was
+// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
+// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
+// everywhere at once.
+//
+// What tells the two apart is which build of each module the machine was last sent, kept with every
+// send. A machine any of whose modules would move to a build its policy or an open plan holds back
+// is not sent by a push that did not name it; the push says which, and why, and how to send it.
+
+// heldBack is why a push that did not name a machine must not send it, empty when it may.
+//
+// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
+// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
+// machine was last sent — including a module the machine was never sent at all. A move is held when
+// the module's policy records rather than rolls out, or when an open plan has not yet sent this
+// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
+// is not known, so a held upgrade cannot be told from anything else.
+func heldBack(node string, modules []string, sent map[string]string, known bool,
+	current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
+	if !known {
+		return []string{"which builds it was last sent is not known — it was last sent before the " +
+			"mesh kept them, or sent a declaration by hand"}
+	}
+	var why []string
+	for _, m := range modules {
+		now := current[m]
+		was, carried := sent[m]
+		if carried && was == now.Commit {
+			continue
+		}
+		move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
+		if !now.RollOut {
+			why = append(why, move+", which its upgrade policy records rather than rolls out")
+			continue
+		}
+		if id := planStillToSend(plans, m, node); id != "" {
+			why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
+		}
+	}
+	sort.Strings(why)
+	return why
+}
+
+// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
+// one holding the module that has neither finished sending it nor sent it here first, and has not
+// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
+// per machine.
+func planStillToSend(plans []inventory.Plan, module, node string) string {
+	for _, p := range plans {
+		s, holds := p.Modules[module]
+		if !p.Open() || !holds {
+			continue
+		}
+		if s == nil {
+			return p.ID
+		}
+		if s.SentAt != nil || s.State == "failed" {
+			continue
+		}
+		first := false
+		for _, n := range s.First {
+			if n == node {
+				first = true
+			}
+		}
+		if !first {
+			return p.ID
+		}
+	}
+	return ""
+}
+
+func fromBuild(was string, carried bool) string {
+	if !carried {
+		return "(never sent it) "
+	}
+	return "from " + buildName(was) + " "
+}
+
+func buildName(commit string) string {
+	if commit == "" {
+		return "a build with no source"
+	}
+	return shortCommit(commit)
+}
+
+// heldMachines reads, for each machine named, why a push that did not name it must not send it
+// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
+// to the send, which says why.
+func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
+	out := map[string][]string{}
+	if len(names) == 0 {
+		return out, nil
+	}
+	inv := open.inventory
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		return nil, err
+	}
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range names {
+		plan, _, err := planFor(ctx, open, node)
+		if err != nil {
+			continue
+		}
+		modules := make([]string, 0, len(plan.Modules))
+		for _, m := range plan.Modules {
+			modules = append(modules, m.Module)
+		}
+		sent, known, err := inv.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
+			out[node] = why
+		}
+	}
+	return out, nil
+}
+
+// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
+// was not sent, that whatever else it is owed waits with it, and the command that sends it.
+func sayHeld(w io.Writer, node string, why []string) {
+	fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
+		"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
+		"grant from this push among it — waits with it. `push %s` sends it\n",
+		node, strings.Join(why, "; "), node)
+}
+
+// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
+// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
+// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
+//
+// `handled` is every machine already sent or already said; it is not considered again. Answers the
+// machines that could not be composed, as refusals.
+func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
+	compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
+	w io.Writer) ([]string, error) {
+	inv := open.inventory
+	var refusals []string
+	// Bounded by the node count: a node is marked handled the round it is considered and is never
+	// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
+	// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
+	// cascade legitimately still converging, so it is said rather than passed over in silence.
+	rounds := 0
+	for {
+		would, err := wouldSend(ctx, open, nodes)
+		if err != nil {
+			return refusals, err
+		}
+		behind, err := inv.Waiting(ctx, would)
+		if err != nil {
+			return refusals, err
+		}
+		var also []string
+		for _, m := range behind {
+			if !handled[m.Node] {
+				also = append(also, m.Node)
+			}
+		}
+		if len(also) == 0 {
+			return refusals, nil
+		}
+		if rounds++; rounds > len(nodes) {
+			fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
+				"should not happen; run `push --behind` to finish\n",
+				rounds-1, strings.Join(also, ", "))
+			return refusals, nil
+		}
+		sort.Strings(also)
+		held, err := heldMachines(ctx, open, also)
+		if err != nil {
+			return refusals, err
+		}
+		var sending []string
+		for _, name := range also {
+			// Every candidate this round is marked handled — the sent ones so they are not
+			// re-listed, the held ones because they stay held, and the refused ones so a machine
+			// that cannot be composed does not make the loop spin on it for ever.
+			handled[name] = true
+			if why, isHeld := held[name]; isHeld {
+				sayHeld(w, name, why)
+				continue
+			}
+			sending = append(sending, name)
+		}
+		if len(sending) == 0 {
+			continue
+		}
+		fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
+			"declaration since changed; sending it too\n", strings.Join(sending, ", "))
+		// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
+		// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
+		// Held for this round only, and after the last round's were given back, so two pushes
+		// cascading into each other's machines never each wait on the other.
+		refused, err := sendRound(ctx, open, sending, compose, d, holder)
+		refusals = append(refusals, refused...)
+		if err != nil {
+			return refusals, err
+		}
+	}
+}
+
+// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
+// is left out of it said, and its declaration allocated.
+func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
+	return func(held context.Context, node string) (sendable, error) {
+		plan, settings, err := planFor(held, open, node)
+		if err != nil {
+			return sendable{}, err
+		}
+		reportUnhostable(node, plan)
+		declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
+		if err == nil {
+			reportLeftOut(node, declared)
+		}
+		return declared, err
+	}
+}

cmd/mesh-controller/held_back_test.go

@@ -0,0 +1,335 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"reflect"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
+)
+
+// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
+// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
+// else that moved is still a consequence the push sends (ADR 0083).
+func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
+	current := map[string]inventory.CurrentBuild{
+		"resolver": {Commit: "c2c2c2c2c2"},
+		"agent":    {Commit: "a2", RollOut: true},
+		"network":  {},
+	}
+	modules := []string{"network", "resolver", "agent"}
+	sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
+
+	// A module whose policy records moved: held, naming it, both builds and why.
+	why := heldBack("laptop", modules, sent, true, current, nil)
+	if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
+		!strings.Contains(why[0], "upgrade policy records") {
+		t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
+	}
+
+	// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
+	sent["resolver"] = "c2c2c2c2c2"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
+		t.Fatalf("a machine whose builds are all current was held: %v", why)
+	}
+
+	// A module whose policy rolls out moved, and no plan holds it: sent, as before.
+	sent["agent"] = "a1"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
+		t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
+	}
+
+	// The last send's builds are not known: held whole.
+	if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
+		!strings.Contains(why[0], "not known") {
+		t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
+	}
+
+	// A module the machine was never sent, under a recording policy: held, and said so.
+	delete(sent, "resolver")
+	sent["agent"] = "a2"
+	if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
+		!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
+		t.Fatalf("a module never sent under a recording policy: %v", why)
+	}
+}
+
+// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
+// naming some other machine must not send them for it.
+func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
+	at := time.Now()
+	current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
+	sent := map[string]string{"agent": "a1"}
+	waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+		"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
+
+	why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
+	if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
+		t.Fatalf("a machine the plan has not reached was not held: %v", why)
+	}
+	// The first machine itself was sent by the plan: not held by it.
+	if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
+		t.Fatalf("the plan's first machine was held: %v", why)
+	}
+	// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
+	for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
+		plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
+			Modules: map[string]*inventory.PlanModule{"agent": s}}}
+		if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
+			t.Errorf("%s: not held: %v", what, why)
+		}
+	}
+	// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
+	for what, plans := range map[string][]inventory.Plan{
+		"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
+		"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "failed"}}}},
+		"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "built"}}}},
+	} {
+		if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
+			t.Errorf("%s: held: %v", what, why)
+		}
+	}
+}
+
+// A send records the build of each module it carried; a module left out of it keeps the build it
+// was last sent, since the machine keeps that one.
+func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
+	current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
+	got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
+		current, map[string]string{"a": "a1", "b": "b1"})
+	if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("carried %v, wanted %v", got, want)
+	}
+	// Not known before: the left-out module is not recorded at all, so it reads as never sent.
+	got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
+	if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("carried %v, wanted %v", got, want)
+	}
+}
+
+// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
+// reads the machine as current — and writes down which machines it declared.
+type recordedDelivery struct {
+	inv      *inventory.Inventory
+	declared []string
+}
+
+func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
+
+func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
+	r.declared = append(r.declared, s.node)
+	return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
+}
+
+// aResolver is a module built from a repository, at a commit, with something on the machine that
+// says which build it is.
+func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
+	t.Helper()
+	m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
+		{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
+	}}
+	if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
+		Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// A third machine on the private network: once it is sent, every other machine's peers change with
+// it, which is a consequence a push must still send — no build moved.
+func aThirdMachine(t *testing.T, open *stores) {
+	t.Helper()
+	ctx := t.Context()
+	record, err := open.inventory.AddNode(ctx, "spare")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
+		t.Fatal(err)
+	}
+	reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
+		{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
+		{"name": "systemd", "present": true}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var profile map[string]any
+	if err := json.Unmarshal(reported, &profile); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
+// naming the anchor, and the laptop — running the same module — left with what it had, by name.
+func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	asked := time.Now().Add(-time.Hour)
+	aResolver(t, open, "c1c1c1c1c1", asked)
+	for _, node := range []string{"anchor", "laptop"} {
+		if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compose := composeForPush(open, gens)
+	d := &recordedDelivery{inv: inv}
+	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
+		t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
+	}
+	digestOfLaptop := func() string {
+		sent, err := inv.Outstanding(ctx, "laptop")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return sent
+	}
+	before := digestOfLaptop()
+
+	// The change merges; the policy is the default, record. `push anchor` sends the anchor...
+	aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
+	d.declared = nil
+	if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	// ...and its cascade leaves the laptop, saying so.
+	var said bytes.Buffer
+	d.declared = nil
+	refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
+	if err != nil || len(refused) != 0 {
+		t.Fatalf("the cascade failed: %v %v", refused, err)
+	}
+	if len(d.declared) != 0 {
+		t.Fatalf("the cascade sent %v a build its policy records", d.declared)
+	}
+	if digestOfLaptop() != before {
+		t.Fatal("the laptop's last send moved: it was sent the held build")
+	}
+	for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
+		"upgrade policy records", "`push laptop` sends it"} {
+		if !strings.Contains(said.String(), want) {
+			t.Errorf("the push did not say %q:\n%s", want, said.String())
+		}
+	}
+
+	// Held and owed something else at once — a peer joined: still not sent, and both said: why it
+	// is held, and that what else it is owed waits with it.
+	aThirdMachine(t, open)
+	d.declared = nil
+	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
+		compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if len(d.declared) != 0 || digestOfLaptop() != before {
+		t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
+	}
+	if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
+		t.Fatalf("the push did not say both:\n%s", said.String())
+	}
+
+	// A policy that rolls out: the laptop is a consequence like any other, and sent.
+	if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
+		t.Fatal(err)
+	}
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
+		compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
+		t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
+	}
+	if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
+		t.Fatalf("the new send did not record the new build: %v", builds)
+	}
+}
+
+// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
+// for another reason is sent by a push that names someone else.
+func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
+	if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
+		t.Fatal(err)
+	}
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compose := composeForPush(open, gens)
+	d := &recordedDelivery{inv: inv}
+	if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// `push spare`, the machine just placed: the others' peers change with it.
+	aThirdMachine(t, open)
+	if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	var said bytes.Buffer
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
+		t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
+	}
+	if strings.Contains(said.String(), "was not sent") {
+		t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
+	}
+
+	// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
+	record, err := inv.NodeByName(ctx, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
+		t.Fatal(err)
+	}
+	d.declared = nil
+	said.Reset()
+	if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
+		t.Fatal(err)
+	}
+	// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
+	// question.
+	if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
+		t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
+	}
+}

cmd/mesh-controller/hold_test.go

@@ -18,16 +18,16 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
 	plain := func(context.Context, string) (sendable, error) {
 		return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
 	}
-	failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
+	failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
-	fine := func(readyNode, []byte) error { return nil }
+	fine := &recordingDelivery{}
 
 	for name, round := range map[string]func() error{
 		"a body that cannot be marshalled": func() error {
-			_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
+			_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
 			return err
 		},
 		"a send that fails": func() error {
-			_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
+			_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
 			return err
 		},
 	} {

cmd/mesh-controller/holdings.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/overlay"
 )
 
 // recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
 	}
 	return said, nil
 }
+
+// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
+// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
+// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
+//
+// **The holders on record, and only the sole claimant when there are none** — the same answer the
+// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
+// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
+// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
+// listed here. A holder off the private network is left out: a resolver named at an address nothing
+// answers is a lookup that waits out its timeout on every name.
+func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
+	var replicated []catalogue.Seat
+	for _, s := range catalogue.Seats() {
+		if s.Replicated && s.Scope == catalogue.ScopeMesh {
+			replicated = append(replicated, s)
+		}
+	}
+	if len(replicated) == 0 {
+		return nil, nil
+	}
+	recorded, err := inv.Holdings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	places, err := onTheNetwork(ctx, inv, shelf)
+	if err != nil {
+		return nil, err
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		address[p.Name] = p.Address
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]map[string]string{}
+	for _, seat := range replicated {
+		var nodes []string
+		for _, h := range recorded {
+			if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
+				nodes = append(nodes, h.Node)
+			}
+		}
+		if len(nodes) == 0 {
+			var derived []string
+			for _, e := range entries {
+				for _, c := range e.Manifest.Claims {
+					if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
+						derived = append(derived, e.On...)
+					}
+				}
+			}
+			if len(derived) == 1 {
+				nodes = derived
+			}
+		}
+		at := map[string]string{}
+		for _, n := range nodes {
+			if address[n] != "" {
+				at[overlay.InternalName(n)] = address[n]
+			}
+		}
+		out[seat.Name] = at
+	}
+	return out, nil
+}

cmd/mesh-controller/issue_204_test.go

@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
 	}
 	cancel() // the sender is going away: its context is cancelled between the send and the record
 	body := []byte(`{"declaration":1,"resources":[]}`)
-	digest, err := recordSent(ctx, inv, "anchor", body)
+	digest, err := recordSent(ctx, inv, "anchor", body, nil)
 	if err != nil {
 		// NodeByName on the cancelled context may itself refuse; the record must still be possible
 		// through the detached context, so look the node up again on a live one.

cmd/mesh-controller/main.go

@@ -73,6 +73,23 @@ func run() error {
 		return askCommand(ctx, args[1:])
 	case "builds":
 		return buildsCommand(ctx, args[1:])
+	// The build queue, controlled by hand (novox/hq ADR 0219).
+	case "queue":
+		return queueCommand(ctx, args[1:])
+	case "cancel":
+		return cancelCommand(ctx, args[1:])
+	case "clear":
+		return clearCommand(ctx, args[1:])
+	case "rebuild":
+		return rebuildCommand(ctx, args[1:])
+	case "replay":
+		return replayCommand(ctx, args[1:])
+	case "kill":
+		return killCommand(ctx, args[1:])
+	case "pause", "resume":
+		return pauseCommand(ctx, args[0], args[1:])
+	case "collection":
+		return collectionCommand(ctx, args[1:])
 	case "plans":
 		return plansCommand(ctx, args[1:])
 	case "pin":
@@ -177,10 +194,11 @@ func usage() {
   seats [--json]                 every seat this mesh defines, what it delivers, and who holds it
   seat rename <from> <to>        rename a seat; its former name still resolves (ADR 0122)
   seat <name> --to <node>/<module>   hand a seat to that assignment as one act; never empty in between (ADR 0131)
+  seat <name> --add <node>/<module>  add a holder beside the others, for a replicated seat (ADR 0223)
   board [--listen ADDR]          the same three questions, as a page that holds nothing
   api --issuer URL [--listen A]  assign and unassign over http, for a surface that is not here
-  assign <node> <module>         put a module on a node
+  assign <node> <module>...      put modules on a node, judged together (ADR 0207)
-  unassign <node> <module>       take it off
+  unassign <node> <module>...    take them off
   take <node> <module>           preview a module's cutover on an adopted node: what runs beside
                                  what it declares; --yes <digest> cuts it over as previewed
   converge <node> [--yes <digest>] [--filter nftables]   preview, then make, an adopted node converged
@@ -200,6 +218,15 @@ func usage() {
   build --behind                 build every module the mesh holds older than its source
   build --on <module>            rebuild every module that stands on this module's artifacts, bases first
   builds [<module>]              what has been built lately, and what came of it
+  queue [--json]                 every ask in the build queue: waiting, in flight (where, how long), dead
+  cancel <id>                    drop a waiting or dead ask; recorded failed, cancelled by hand
+  clear [--dead]                 cancel every waiting ask (and the dead ones); never one in flight
+  rebuild <module|build-id>      ask the module's source again, or that build's, under a new id
+  replay <build-id> [--register [--older]]   that build's commit again; a dry run unless --register
+  kill <id>                      end a build where it runs; recorded failed, killed by hand
+  pause [<node>] / resume [<node>]   the build seat's holder there, or every holder, takes nothing new / again
+  plans retry <id>               ask a failed plan's failed builds again, and carry the plan on
+  collection [--json]            kept archives held/unheld by a manifest, and what the sweep may let go
   builder issue <name>           a broker account for a build machine, scoped to build work,
                                  delivered as the builder module's broker secret (module add it first)
   licence add|list|use|key       model access, under the name a person calls it
@@ -253,6 +280,13 @@ func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
 // registered, the same as the waiting command does. Said either way, so the daemon's log tells what
 // became of a build nobody was watching.
 func (b builds) Built(ctx context.Context, result link.BuildResult) error {
+	// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
+	// unmerged branch was heard here like any build, registered, and its definition reached a machine
+	// before anyone had reviewed it.
+	if result.DryRun {
+		fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
+		return nil
+	}
 	manifest, _, err := takeIn(ctx, b.inv, result)
 	// When it was asked, so a plan takes as its outcome only a build asked for it or after it
 	// (novox/hq 04-ISSUES/219). Zero when the id does not say.
@@ -261,7 +295,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 	case err != nil && result.Failed != "":
 		fmt.Printf("%s: %v\n", result.ID, err)
 		if result.Module != "" {
-			planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked)
+			planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
 		} else {
 			planFailedBuild(ctx, b.open, result)
 		}
@@ -270,18 +304,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 		// Not a failure: the module is already at what a later request built. A plan that asked
 		// before that later request is answered by it; one that asked after it ignores this.
 		fmt.Printf("%s: %v\n", result.ID, err)
-		planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
+		planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
 		return nil
 	case err != nil:
 		fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
 		if manifest.Module != "" {
-			planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked)
+			planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
 		}
 		return nil
 	}
 	fmt.Printf("%s: %s %s registered, built on %s from %s\n",
 		result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
 	saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
-	planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
+	planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
 	return nil
 }

cmd/mesh-controller/modules.go

@@ -334,8 +334,10 @@ func moduleCommand(ctx context.Context, args []string) error {
 }
 
 func assignCommand(ctx context.Context, verb string, args []string) error {
-	if len(args) != 2 {
+	// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
-		return fmt.Errorf("%s <node> <module>", verb)
+	// service manager and the package manager — can only go on, or come off, together.
+	if len(args) < 2 {
+		return fmt.Errorf("%s <node> <module> [<module>…]", verb)
 	}
 	open, err := openStores(ctx)
 	if err != nil {
@@ -349,7 +351,7 @@ func assignCommand(ctx context.Context, verb string, args []string) error {
 	if verb == "unassign" {
 		act = unassign
 	}
-	said, err := act(ctx, open, args[0], args[1])
+	said, err := act(ctx, open, args[0], args[1:]...)
 	if said != "" {
 		fmt.Println(said)
 	}
@@ -659,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
 	// durable subscription nobody reads.
 	if consumer, needed := broker.ConsumerFor(broker.Principal{
 		Kind: broker.KindModule, Node: node, Module: m.Module,
-		Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
+		Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
 	}); needed {
 		if busAddress == "" {
 			fmt.Printf("  %s consumes; its consumer is created when the bus is reachable (`push`, then "+

cmd/mesh-controller/network.go

@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
 	if err != nil {
 		return nil, err
 	}
+	// No registry trust is composed here any more: the container runtime's module states it, told
+	// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
 	g, err := overlay.From(nodes, cidr, "")
-	if g != nil {
-		// The artifact store, as this network reaches it. Found rather than configured: the
-		// provider is whichever module offers it, on whichever machine holds that module — and if
-		// nothing does yet (genesis raises the registry before the catalogue knows it), there is
-		// no trust to write and nothing is written (novox/hq ADR 0082).
-		//
-		// Refused rather than composed without it when the question could not be answered: a
-		// declaration missing the trust because a lookup failed is a machine that cannot pull,
-		// delivered by a push that reported success — and nothing recomposes it until the next
-		// push (the shape of novox/hq issues 042/048, reappearing as a race).
-		at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
-		if storeErr != nil {
-			return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
-		}
-		if found {
-			g.TrustRegistry(overlay.InternalName(at) + ":" + port)
-		}
-	}
 	if err != nil && len(refused) > 0 {
 		// The network is missing something, and some machines could not be resolved at all. Those
 		// are almost always the same fact: a node that does not resolve contributes nothing, so

cmd/mesh-controller/network_test.go

@@ -254,11 +254,10 @@ func theResolver(t *testing.T) catalogue.Manifest {
 	return m
 }
 
-// The resolver is handed every machine on the private network as a wildcard, the same set and the
+// The resolver is handed every machine on the private network as a wildcard, and is handed it again
-// same source as the hosts file, and is handed it again when a machine leaves — through the
+// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
-// module's own manifest asking for the fact, with no module of the mesh's own in between (hal
+// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
-// dnsmasq-app conversion, novox/hq 08-connectivity). The runtime on that machine is pointed at the
+// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
-// machine's own address, where the resolver answers for its containers.
 func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
@@ -293,11 +292,12 @@ func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *t
 			t.Errorf("the resolver's machines lack %q:\n%s", want, first)
 		}
 	}
+	// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
+	// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
+	// container asks, read only when the runtime starts.
 	for _, r := range composed(t, open, "anchor").Resources {
 		if r["id"] == "dnsmasq.runtime-dns" {
-			if !strings.Contains(r["content"].(string), `"10.77.0.1"`) || r["into"] != "json" {
+			t.Errorf("the resolver still writes the runtime's own dns: %v", r)
-				t.Errorf("the runtime is not pointed at this machine's own address, written into its file: %v", r)
-			}
 		}
 	}
 

cmd/mesh-controller/nodes.go

@@ -386,8 +386,12 @@ func brokerCommand(ctx context.Context, args []string) error {
 	if len(args) > 0 && args[0] == "accounts" {
 		return busAccounts(ctx, args[1:])
 	}
+	if len(args) > 0 && args[0] == "consumer-reset" {
+		return consumerReset(args[1:])
+	}
 	if len(args) == 0 || args[0] != "show" {
-		return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file>")
+		return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
+			"broker consumer-reset <stream> <consumer>")
 	}
 	known, err := broker.FromEnvironment()
 	if errors.Is(err, broker.ErrNotConfigured) {
@@ -407,6 +411,34 @@ func brokerCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
+// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
+// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
+func consumerReset(args []string) error {
+	if len(args) != 2 {
+		return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
+	}
+	address, err := broker.BusAddress()
+	if err != nil {
+		return err
+	}
+	js, err := broker.Dial(address)
+	if err != nil {
+		return fmt.Errorf("cannot reach the bus: %w", err)
+	}
+	defer js.Close()
+	before, after, err := js.ResetConsumer(args[0], args[1])
+	if err != nil {
+		return err
+	}
+	fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
+	fmt.Printf("  before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
+		before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
+	fmt.Printf("  after:  delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
+		"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
+	return nil
+}
+
 // heardFrom says when a node was last heard from, in a form somebody can act on.
 //
 // "never" and "an hour ago" are different answers and are kept different. A node that has never
@@ -473,6 +505,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		fmt.Printf("  public domain     %s\n", domain)
 	}
 
+	// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
+	// capabilities, because it is something not working now and they are a description.
+	failing, err := failingProviders(ctx, inv)
+	if err != nil {
+		return err
+	}
+	if here := failingOn(failing, name); len(here) > 0 {
+		fmt.Printf("\n  %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
+		for _, line := range failingLines(here, time.Now()) {
+			fmt.Printf("  %s\n", line)
+		}
+	}
+
 	held, err := inv.Profile(ctx, name)
 	if err != nil {
 		return err

cmd/mesh-controller/order_test.go

@@ -147,6 +147,14 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
 		{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
 		{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
 		{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
+		// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
+		// shows it is one — and a directory that may be shared code is still shared.
+		{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
+		{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
+		{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
+		{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
+		{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
+		{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
 	} {
 		if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
 			t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)

cmd/mesh-controller/plan.go

@@ -8,8 +8,10 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"slices"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
@@ -127,6 +129,7 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 		// a mesh-wide gatherer may pass over — see notResolvable.
 		return catalogue.Resolution{}, nil, notResolvable{err}
 	}
+	logUnheld(nodeName, resolved.Unheld)
 
 	// The credential for each thing this node takes from elsewhere. Made once and kept, so the
 	// password a provider is told to create is the one its consumer was given — and sealed to
@@ -394,9 +397,49 @@ func declarationWith(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return sendable{}, err
 	}
-	return sendable{Resources: composed.Resources, Adoption: adoption,
+	out := sendable{Resources: composed.Resources, Adoption: adoption,
-		Received: composed.Received, Mesh: with.Mesh,
+		Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
-		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
+		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}
+	// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
+	// 0221). Read only on the send path: a question about what would be sent records nothing.
+	if choosing == Allocating {
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return sendable{}, err
+		}
+		before, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return sendable{}, err
+		}
+		if !known {
+			before = nil
+		}
+		names := make([]string, 0, len(plan.Modules))
+		for _, m := range plan.Modules {
+			names = append(names, m.Module)
+		}
+		out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
+	}
+	return out, nil
+}
+
+// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
+// issue 259): the module's current build for each module in it, and for a module left out of it
+// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
+// that is not known. Never nil, so a send through here always records what it knows.
+func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
+	before map[string]string) map[string]string {
+	out := map[string]string{}
+	for _, m := range modules {
+		if _, left := leftOut[m]; left {
+			if was, kept := before[m]; kept {
+				out[m] = was
+			}
+			continue
+		}
+		out[m] = current[m].Commit
+	}
+	return out
 }
 
 // sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
@@ -657,6 +700,20 @@ func renderingFor(ctx context.Context, open *stores, node string,
 		machines[name] = at
 	}
 
+	// And every zone a module in the mesh answers itself (novox/hq ADR 0199), for the mesh's resolver
+	// to forward.
+	zones, err := zonesInTheMesh(ctx, open)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
+	// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
+	// lists every holder of the mesh's resolver.
+	replicas, err := replicatedHolders(ctx, inv, shelf)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
+
 	// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
 	// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
 	// before it had an address on the private network. A machine joins through the tunnel now, and
@@ -722,18 +779,90 @@ func renderingFor(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
+	// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
+	// 0222): the artifact store as this network reaches it, which the container runtime is told to
+	// trust (ADR 0082). The same address composed into every reference the mesh built.
+	var reach map[string]string
+	if artifactStore != "" {
+		reach = map[string]string{"mesh-artifact-store": artifactStore}
+	}
 	return catalogue.Rendering{
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Machines: machines,
+		Machines: machines, Zones: zones, Holders: replicas,
-		Suffix:   overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
+		Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
 		Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
-		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
+		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
 		BusUsers: busUsers,
 	}, record, nil
 }
 
+// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
+// 0199): the zone settled from that node's settings, the node's private address, the port the
+// answering listen is published on there.
+//
+// Read across every machine's resolution, as the roster once read routed names: a node whose set does
+// not compose declares nothing and is passed over, so one broken machine does not cost the rest their
+// zones; a store that cannot be read is raised, naming the machine, because returning the zones
+// without it would withdraw them from the resolver as if the operator had (novox/hq 04-ISSUES/152).
+// What the mesh refuses about the zones together — one declared twice, one shadowing the mesh's
+// suffix or a node's public domain — is refused here, by name.
+func zonesInTheMesh(ctx context.Context, open *stores) ([]catalogue.ZoneAt, error) {
+	inv := open.inventory
+	places, err := inv.Overlays(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("where the machines are cannot be read: %w", err)
+	}
+	address := map[string]string{}
+	for _, p := range places {
+		if strings.TrimSpace(p.Address) != "" {
+			address[p.Name] = p.Address
+		}
+	}
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("which machines the mesh has cannot be read: %w", err)
+	}
+	var zones []catalogue.ZoneAt
+	var public []string
+	for _, n := range nodes {
+		plan, _, err := planFor(ctx, open, n.Name)
+		switch {
+		case unresolvable(err):
+			continue
+		case err != nil:
+			return nil, fmt.Errorf("the zones %s answers cannot be read: %w", n.Name, err)
+		}
+		if plan.PublicDomain != "" {
+			public = append(public, plan.PublicDomain)
+		}
+		for _, m := range plan.Modules {
+			if m.Zone == nil {
+				continue
+			}
+			at := address[n.Name]
+			if at == "" {
+				// Not on the private network yet: nothing could reach its answerer.
+				continue
+			}
+			published, layers, err := portsGivenOn(ctx, inv, n.Name, m)
+			if err != nil {
+				return nil, fmt.Errorf("the zone %s declares on %s cannot be read: %w", m.Module, n.Name, err)
+			}
+			z, err := catalogue.ZoneOn(m, layers, published, n.Name, at)
+			if err != nil {
+				return nil, err
+			}
+			zones = append(zones, *z)
+		}
+	}
+	if problems := catalogue.ZonesProblems(zones, overlay.Suffix(), public); len(problems) > 0 {
+		return nil, fmt.Errorf("the mesh's zones cannot be forwarded:\n  - %s", strings.Join(problems, "\n  - "))
+	}
+	return zones, nil
+}
+
 // certificateFor is what the mesh certifies about one machine's internal name.
 //
 // It reaches across two contexts and reads neither one's store from the other: `inventory` knows
@@ -1266,6 +1395,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
 	//
 	// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
 	// resource of that module, so the question is whether it is here.
+	list, missing, err := busUserList(ctx, inv, onThisNode)
+	if len(missing) > 0 {
+		fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
+			"for: %s. Each is a user that cannot connect until one is issued\n",
+			len(missing), strings.Join(missing, ", "))
+	}
+	return list, err
+}
+
+// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
+// for want of a credential. Asked on every send to decide whether the machine holding the bus must
+// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
+func busUserList(ctx context.Context, inv *inventory.Inventory,
+	onThisNode []catalogue.Manifest) (string, []string, error) {
 	holdsTheBus := false
 	for _, m := range onThisNode {
 		if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
@@ -1273,37 +1416,33 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
 		}
 	}
 	if !holdsTheBus {
-		return "", nil
+		return "", nil, nil
 	}
 
 	records, err := inv.BusRecords(ctx)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	users, err := broker.Users(records)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	kept, err := inv.BusUsers(ctx)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	hashes := make(map[string]string, len(kept))
 	for name, u := range kept {
 		hashes[name] = u.PasswordHash
 	}
 	filled, missing := broker.WithPasswords(users, hashes)
-	if len(missing) > 0 {
-		fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
-			"for: %s. Each is a user that cannot connect until one is issued\n",
-			len(missing), strings.Join(missing, ", "))
-	}
 	if len(filled) == 0 {
-		return "", fmt.Errorf(
+		return "", missing, fmt.Errorf(
 			"this machine runs the bus and not one user has a credential, so the composed list " +
 				"would refuse every connection in the mesh")
 	}
-	return broker.ComposeAccounts(filled)
+	list, err := broker.ComposeAccounts(filled)
+	return list, missing, err
 }
 
 // providerModuleOf is which module answers a need on the providing node: the one in this node's
@@ -1325,3 +1464,90 @@ func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.C
 	}
 	return ""
 }
+
+// unheldLogged is what was last logged about each node's unmet seat dependencies, so the log says
+// each change once (novox/hq ADR 0207), on stderr so `status --json` stays a document.
+//
+// **The serving controller's log only.** planFor runs for every node on every push, assignment and
+// status — `blockedElsewhere` alone resolves the whole mesh — and a one-shot command starts with an
+// empty memory, so every node's report was "a change" and a push printed the whole mesh's list,
+// burying the line about the node it acted on. A command says what concerns its own act instead
+// (unheldChange, reportUnheldPushed); the full list is `status`'s.
+var (
+	unheldLogged     = map[string]string{}
+	unheldLoggedMu   sync.Mutex
+	logUnheldChanges bool
+)
+
+// logUnheld logs a node's unmet seat dependencies when they differ from what was last logged for
+// it, including when they become none — in the serving controller, and nowhere else.
+func logUnheld(node string, unheld []catalogue.Unheld) {
+	if !logUnheldChanges {
+		return
+	}
+	lines := make([]string, 0, len(unheld))
+	for _, u := range unheld {
+		lines = append(lines, u.String())
+	}
+	now := strings.Join(lines, "\n")
+	unheldLoggedMu.Lock()
+	before, seen := unheldLogged[node]
+	unheldLogged[node] = now
+	unheldLoggedMu.Unlock()
+	if (seen && before == now) || (!seen && now == "") {
+		return
+	}
+	if now == "" {
+		fmt.Fprintf(os.Stderr, "%s: every seat its modules depend on is held (novox/hq ADR 0207)\n", node)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "%s: %d unmet seat dependenc(ies), reported and not refused (novox/hq ADR 0207):\n  %s\n",
+		node, len(lines), strings.Join(lines, "\n  "))
+}
+
+// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
+// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
+// which includes every one of a module just assigned — and each now met that was not. Nothing about
+// any other node, and nothing that was already true before the act.
+func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
+	judge := func(names []string) map[string]catalogue.Unheld {
+		var set []catalogue.Manifest
+		for _, n := range names {
+			if m, known := shelf[n]; known {
+				set = append(set, m)
+			}
+		}
+		out := map[string]catalogue.Unheld{}
+		for _, u := range catalogue.UnheldDependencies(shelf, node, set, nil) {
+			out[u.Module+" "+u.Seat] = u
+		}
+		return out
+	}
+	was, now := judge(before), judge(after)
+	var lines []string
+	for _, k := range sortedNames(now) {
+		if _, already := was[k]; !already {
+			lines = append(lines, "but "+now[k].String())
+		}
+	}
+	for _, k := range sortedNames(was) {
+		if _, still := now[k]; still {
+			continue
+		}
+		u := was[k]
+		if !slices.Contains(after, u.Module) {
+			continue // went with its module, which says nothing about the seat
+		}
+		lines = append(lines, fmt.Sprintf("and %s on %s now has %s held", u.Module, node, u.Seat))
+	}
+	return lines
+}
+
+func sortedNames[V any](m map[string]V) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}

cmd/mesh-controller/plan_retry.go

@@ -0,0 +1,401 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A plan follows what is done to the build queue (novox/hq ADR 0219).
+//
+// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
+//
+//   - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
+//     is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
+//     sends a reader looking for a fault that is a decision. It says what it waits on instead.
+//   - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
+//     failed plan's failed modules and the plan goes on from that tier as if they had built the
+//     first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
+//     beside it — beside it, the plan would either ask it again or stay failed on an outcome the
+//     rebuild has already replaced.
+
+// pauseView is whether the build seat takes work: the holders that said they are paused, and
+// whether that is every holder.
+type pauseView struct {
+	Nodes []string
+	All   bool
+}
+
+// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
+// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
+func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
+	if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
+		return "", false
+	}
+	var asked *time.Time
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
+			if asked == nil || s.AskedAt.Before(*asked) {
+				asked = s.AskedAt
+			}
+		}
+	}
+	if asked == nil {
+		return "", false
+	}
+	waited := now.Sub(*asked)
+	said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
+		strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
+	// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
+	// longer than a day is named.
+	if waited > pausedTooLong {
+		said += " — PAUSED OVER A DAY: `resume` takes builds again"
+	}
+	return said, true
+}
+
+// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
+const pausedTooLong = 24 * time.Hour
+
+// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
+// where the seat being paused changes what a plan says.
+func awaitsABuild(plans []inventory.Plan) bool {
+	for _, p := range plans {
+		if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
+			continue
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if s := p.Modules[m]; s != nil && s.State == "asked" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
+// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
+// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
+// reads as late, which is what it said before this existed.
+func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
+	if !awaitsABuild(plans) {
+		return pauseView{}
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return pauseView{}
+	}
+	seat := buildSeatAmong(entries)
+	holders := holdersAmong(entries, seat)
+	if len(holders) == 0 {
+		return pauseView{}
+	}
+	address, err := broker.BusAddress()
+	if err != nil {
+		return pauseView{}
+	}
+	js, err := broker.Dial(address)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
+		return pauseView{}
+	}
+	defer js.Close()
+	said, err := link.PausedSaid(js, seat, holders)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
+		return pauseView{}
+	}
+	return pauseOf(holders, said)
+}
+
+// pauseOf is the view from what each holder said.
+func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
+	var v pauseView
+	for _, n := range holders {
+		if said[n].Paused {
+			v.Nodes = append(v.Nodes, n)
+		}
+	}
+	sort.Strings(v.Nodes)
+	v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
+	return v
+}
+
+// failedIn is the modules of a plan's current tier that failed to build, sorted.
+func failedIn(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "failed" {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
+// that holds what this one held now (issue 254, ADR 0218).
+func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
+	for _, q := range plans {
+		if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
+			(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
+			continue
+		}
+		return q, true
+	}
+	return inventory.Plan{}, false
+}
+
+// retryRefusal is why a plan cannot be retried, or nothing.
+func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
+	switch {
+	case p.State == inventory.PlanDone:
+		return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
+	case p.State == inventory.PlanSuperseded:
+		return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
+	case p.Open():
+		return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
+	}
+	if len(failedIn(p)) > 0 {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		return nil
+	}
+	stopped := stoppedRollouts(p)
+	if len(stopped) == 0 {
+		return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
+			p.Tier, p.ID, p.Note)
+	}
+	// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
+	// sent — a newer build, and sending this one again would put the older build back on its machines.
+	for _, m := range stopped {
+		if q, found := newerPlanFor(m, p, plans); found {
+			return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
+				"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
+		}
+	}
+	return nil
+}
+
+// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
+// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
+func stoppedRollouts(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
+			len(s.First) > 0 && s.Why != "" {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
+func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
+	for _, q := range plans {
+		if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
+			continue
+		}
+		for _, tier := range q.Tiers {
+			for _, m := range tier {
+				if m == module {
+					return q, true
+				}
+			}
+		}
+	}
+	return inventory.Plan{}, false
+}
+
+// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
+// variable so a test of a retried rollout needs no machine.
+var sendRollout = sendToEach
+
+// resumed sets a failed plan building again once nothing in its tier is failed.
+func resumed(p *inventory.Plan, why string) {
+	if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
+		p.State = inventory.PlanBuilding
+		p.Note = why
+	}
+}
+
+// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
+// that tier: what follows is the plan going on as if they had built the first time.
+func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
+	inv := open.inventory
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return "", err
+	}
+	defer release()
+	p, err := inv.PlanByID(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return "", err
+	}
+	recent, err := inv.RecentPlans(ctx, 50)
+	if err != nil {
+		return "", err
+	}
+	plans = append(plans, recent...)
+	if err := retryRefusal(p, plans); err != nil {
+		return "", err
+	}
+	if len(failedIn(p)) == 0 {
+		return retryRollouts(ctx, open, &p)
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return "", err
+	}
+	byName := map[string]inventory.Entry{}
+	for _, e := range entries {
+		byName[e.Manifest.Module] = e
+	}
+	failed := failedIn(p)
+	var asked []string
+	for _, m := range failed {
+		askModule(ctx, &p, m, byName)
+		if s := p.Modules[m]; s.State == "asked" {
+			asked = append(asked, m+" as "+s.Build)
+		}
+	}
+	resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
+	if err := inv.SavePlan(ctx, p); err != nil {
+		return "", err
+	}
+	if p.State != inventory.PlanBuilding {
+		return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
+	}
+	return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
+		p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
+}
+
+// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
+// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
+func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
+	inv := open.inventory
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return false, "", err
+	}
+	defer release()
+	openPlans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	recent, err := inv.RecentPlans(ctx, 20)
+	if err != nil {
+		return false, "", err
+	}
+	p, found := planHolding(module, openPlans, recent)
+	if !found {
+		return false, "", nil
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	byName := map[string]inventory.Entry{}
+	for _, e := range entries {
+		byName[e.Manifest.Module] = e
+	}
+	was := p.State
+	askModule(ctx, &p, module, byName)
+	s := p.Modules[module]
+	resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
+	if err := inv.SavePlan(ctx, p); err != nil {
+		return false, "", err
+	}
+	if s.State != "asked" {
+		return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
+	}
+	said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
+		s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
+	switch {
+	case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
+		said += "; the plan had failed and builds again from this tier"
+	case was == inventory.PlanFailed:
+		said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
+	}
+	return true, said, nil
+}
+
+// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
+// yet built, else the newest failed plan whose current tier does, and that no open plan of its
+// repository supersedes.
+func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
+	holds := func(p inventory.Plan) bool {
+		if p.Tier >= len(p.Tiers) {
+			return false
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if m == module {
+				s := p.Modules[m]
+				return s == nil || s.State != "built"
+			}
+		}
+		return false
+	}
+	for _, p := range openPlans {
+		if holds(p) {
+			return p, true
+		}
+	}
+	sorted := append([]inventory.Plan(nil), recent...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
+	for _, p := range sorted {
+		if p.State != inventory.PlanFailed || !holds(p) {
+			continue
+		}
+		if _, superseded := newerOpenPlan(p, openPlans); superseded {
+			continue
+		}
+		return p, true
+	}
+	return inventory.Plan{}, false
+}
+
+// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
+// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
+// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
+func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
+	var said []string
+	for _, m := range stoppedRollouts(*p) {
+		s := p.Modules[m]
+		sent, err := sendRollout(ctx, open, s.First)
+		if err != nil {
+			return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
+				m, strings.Join(s.First, ", "), p.ID, err)
+		}
+		now := time.Now().UTC()
+		s.First, s.FirstAt, s.Why = sent, &now, ""
+		said = append(said, m+" to "+strings.Join(sent, ", "))
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
+	if err := open.inventory.SavePlan(ctx, *p); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
+		"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
+}

cmd/mesh-controller/plans_one_at_a_time_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
+// process both run the plan timer on one store. Only the one holding the plans moves them; the other
+// leaves them alone, and moves them once they are let go.
+func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	now := time.Now().UTC()
+	// Every tier done: the next step is the plan's last, and needs nothing but the store.
+	plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
+		State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
+		Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+
+	// The other controller: its own connections to the same store, holding the plans.
+	other, err := inventory.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(other.Close)
+	release, err := other.HoldPlans(ctx, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(release) // before the close above: a pool waits for a connection still held
+
+	advancePlans(ctx, open)
+	if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
+		t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
+	}
+
+	release()
+	advancePlans(ctx, open)
+	if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
+		t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
+	}
+}

cmd/mesh-controller/push.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"sort"
@@ -62,6 +63,9 @@ func connectLink(ctx context.Context, inv *inventory.Inventory, enroller link.En
 }
 
 func serve(ctx context.Context) error {
+	// The one process whose log is read over time, so the one that says each change to a node's
+	// unmet seat dependencies once (novox/hq ADR 0207).
+	logUnheldChanges = true
 	open, err := openStores(ctx)
 	if err != nil {
 		return err
@@ -128,6 +132,11 @@ func serve(ctx context.Context) error {
 	if err := server.Answers(following{open}); err != nil {
 		return err
 	}
+	// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
+	// 0224): a provider's journal must not be the only place that says so.
+	if err := server.Watches(standings{inv}); err != nil {
+		return err
+	}
 
 	// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
 	// from the store's row, so what the seat declares is what is answered.
@@ -213,8 +222,9 @@ func declare(ctx context.Context, args []string) error {
 	}
 	// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
 	// is still what the machine was last told, and status must not read it as current for the one
-	// the mesh would compose.
+	// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
-	if _, err := recordSent(ctx, inv, node, raw); err != nil {
+	// the mesh did not compose it, so a push that does not name this machine treats it as held.
+	if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
 		return err
 	}
 	fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -312,7 +322,7 @@ func pushCommand(ctx context.Context, args []string) error {
 		if len(needsOne) == 0 {
 			// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
 			// must never look the same.
-			fmt.Println("every machine is doing what it was told")
+			fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
 			return nil
 		}
 	}
@@ -353,6 +363,46 @@ func pushCommand(ctx context.Context, args []string) error {
 		}
 		asked = append(asked, n.Name)
 	}
+	// **A push that named no machine says so, first.** Through the console a machine the caller
+	// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
+	// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
+	// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
+	if len(args) == 0 {
+		which := "every machine"
+		if *behind {
+			which = "every machine that is behind"
+		}
+		fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
+			which, len(asked), strings.Join(asked, ", "))
+	}
+
+	// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
+	// user list, and a module's new grants are refused by the bus until that list says them. Among
+	// the machines asked it goes first; not among them and behind, it is added — a named push whose
+	// module gained a state would otherwise send the code and leave the right to use it for the
+	// cascade below, after.
+	holder, holderBehind, err := brokerBehind(ctx, open, asked)
+	if err != nil {
+		return err
+	}
+	// **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than
+	// named, it is sent its whole declaration, and a build its policy records or a plan has not sent
+	// it yet would go with the user list. Named and left, with what that costs.
+	saidHeld := map[string]bool{}
+	if holderBehind && len(args) == 1 {
+		held, err := heldMachines(ctx, open, []string{holder})
+		if err != nil {
+			return err
+		}
+		if why, isHeld := held[holder]; isHeld {
+			sayHeld(os.Stdout, holder, why)
+			fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+
+				"sent, the bus may refuse what this push's machines were newly granted\n", holder)
+			holderBehind = false
+			saidHeld[holder] = true
+		}
+	}
+	asked = brokerFirst(asked, holder, holderBehind)
 
 	// Held from composing to sending, so a converge on one of them cannot send between the two
 	// and be overtaken by what was composed before it (novox/hq ADR 0100).
@@ -360,6 +410,9 @@ func pushCommand(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
+	// Each machine's unmet seat dependencies (novox/hq ADR 0207), said after the sends: in full
+	// for a machine named, as a count for each of many — the full list is `status`'s.
+	unheld := map[string][]catalogue.Unheld{}
 	sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) {
 		plan, settings, err := planFor(held, open, node)
 		if err != nil {
@@ -369,6 +422,7 @@ func pushCommand(ctx context.Context, args []string) error {
 		// healthy modules beside it are still resolved and sent. Reported so it is not silently
 		// dropped — the remedy is to move it, and until then the rest of the node converges.
 		reportUnhostable(node, plan)
+		unheld[node] = plan.Unheld
 		// The private network is in here with everything else. It used to be composed separately
 		// and prepended, which meant every machine with an address was on it and no machine could
 		// be kept off. It is a module now, so it arrives the way a module does.
@@ -379,34 +433,21 @@ func pushCommand(ctx context.Context, args []string) error {
 		return declared, err
 	})
 
-	sentDigest := map[string]string{}
 	defer release()
-	for _, s := range sending {
+	// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
-		// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
+	// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
-		// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
+	bus := overTheBus{open: open, server: server, signer: ident}
-		body, err := s.declared.Body()
+	sentDigest, err := deliver(ctx, bus, holder, sending)
-		if err != nil {
+	if err != nil {
-			return err
-		}
-		if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
-			return err
-		}
-		// After it is away, not before. A digest recorded for something that failed to send would
-		// make the machine look current for a declaration it never received.
-		digest, err := recordSent(ctx, inv, s.node, body)
-		if err != nil {
-			return err
-		}
-		sentDigest[s.node] = digest
-		fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
-	}
-	release()
-	fmt.Printf("\n%d node(s) told\n", len(sending))
-	// And each machine's memberships, as every other send does (ADR 0160): a push is the one most
-	// operators run, and on 2026-10-01 it was the one path that issued none.
-	if err := issueMemberships(ctx, open, server, sending); err != nil {
 		return err
 	}
+	release()
+	told := make([]string, 0, len(sending))
+	for _, r := range sending {
+		told = append(told, r.node)
+	}
+	fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
+	reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
 
 	// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
 	// issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's
@@ -416,86 +457,21 @@ func pushCommand(ctx context.Context, args []string) error {
 	//
 	// Compared against what each machine was last SENT, not against a before/after of this push:
 	// the mint usually happened at `assign` or `module issue`, before this command ran, so the
-	// only durable signal is "what it should be" versus "what it last received". A machine behind
+	// only durable signal is "what it should be" versus "what it last received". Bounded: a
-	// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
-	// machine was behind and left it so would be the very silence this removes. Bounded: a
 	// flushed send may itself mint, so this converges over a few rounds.
+	//
+	// **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose
+	// modules would move to a build their upgrade policy records rather than rolls out, or that an
+	// open plan has not sent it yet. It is named, with why, and left for a push that names it.
 	if len(args) == 1 {
-		flushed := map[string]bool{args[0]: true}
+		handled := map[string]bool{args[0]: true}
-		// Bounded by the node count: a node is marked flushed the round it is handled and is
+		for n := range saidHeld {
-		// never handled twice, so the loop cannot run more than len(nodes) rounds. The bound is
+			handled[n] = true
-		// a guard against a logic error, not a real limit — if it were ever hit, that is a bug
+		}
-		// rather than a cascade legitimately still converging, so it is said rather than passed
+		refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
-		// over in silence, unlike the earlier fixed cap that could stop a real cascade short.
+		refusals = append(refusals, refused...)
-		rounds := 0
+		if err != nil {
-		for {
+			return err
-			would, err := wouldSend(ctx, open, nodes)
-			if err != nil {
-				return err
-			}
-			behind, err := inv.Waiting(ctx, would)
-			if err != nil {
-				return err
-			}
-			var also []string
-			for _, m := range behind {
-				if !flushed[m.Node] {
-					also = append(also, m.Node)
-				}
-			}
-			if len(also) == 0 {
-				break
-			}
-			if rounds++; rounds > len(nodes) {
-				fmt.Printf("\nstopped cascading after %d rounds with %s still behind — this "+
-					"should not happen; run `push --behind` to finish\n",
-					rounds-1, strings.Join(also, ", "))
-				break
-			}
-			sort.Strings(also)
-			fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
-				"declaration since changed; sending it too\n", strings.Join(also, ", "))
-			// Tolerantly, exactly as the named send above: a machine that cannot be composed is
-			// collected as a refusal and reported at the end, and the others are still sent
-			// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
-			// all-or-nothing — so one swept machine's compose error failed the operator's named
-			// push and skipped its --wait, the very intolerance the main path exists to avoid.
-			// Held for this round only, and after the last round's were given back, so two pushes
-			// cascading into each other's machines never each wait on the other.
-			refused, err := sendRound(ctx, open, also,
-				func(held context.Context, node string) (sendable, error) {
-					plan, settings, err := planFor(held, open, node)
-					if err != nil {
-						return sendable{}, err
-					}
-					reportUnhostable(node, plan)
-					declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
-					if err == nil {
-						reportLeftOut(node, declared)
-					}
-					return declared, err
-				},
-				func(s readyNode, body []byte) error {
-					if err := link.Declare(ctx, server.Bus(), ident, s.node, body,
-						15*time.Second); err != nil {
-						return err
-					}
-					if _, err := recordSent(ctx, inv, s.node, body); err != nil {
-						return err
-					}
-					fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
-					return nil
-				})
-			refusals = append(refusals, refused...)
-			if err != nil {
-				return err
-			}
-			// Every candidate this round is marked handled — the sent ones so they are not
-			// re-listed, and the refused ones so a machine that cannot be composed does not make
-			// the loop spin on it for ever. Its refusal is already in the report.
-			for _, name := range also {
-				flushed[name] = true
-			}
 		}
 	}
 
@@ -612,10 +588,10 @@ func composeEach(names []string, allot func(node string) (int64, error),
 // sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
 // back on every way out — a body that cannot be marshalled and a send that fails included
 // (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
-// still sent.
+// still sent. Their memberships go before their declarations, as every send's do (issue 249).
 func sendRound(ctx context.Context, open *stores, names []string,
 	compose func(held context.Context, node string) (sendable, error),
-	send func(s readyNode, body []byte) error) ([]string, error) {
+	d delivery, holder string) ([]string, error) {
 	held, release, err := holdNodes(ctx, open, names)
 	if err != nil {
 		return nil, err
@@ -624,18 +600,246 @@ func sendRound(ctx context.Context, open *stores, names []string,
 	sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
 		return compose(held, node)
 	})
-	for _, s := range sending {
+	if _, err := deliver(held, d, holder, sending); err != nil {
-		body, err := s.declared.Body()
+		return refused, err
-		if err != nil {
-			return refused, err
-		}
-		if err := send(s, body); err != nil {
-			return refused, err
-		}
 	}
 	return refused, nil
 }
 
+// delivery is the two acts of sending machines what they should be, apart, so the order between
+// them is one function's and can be read and tested there (novox/hq issue 249).
+type delivery interface {
+	// grant issues what the machines' modules may do — each module's state raised and its
+	// membership issued — for every machine about to be sent.
+	grant(ctx context.Context, sending []readyNode) error
+	// declare sends one machine its declaration and records it sent, answering the digest.
+	declare(ctx context.Context, s readyNode, body []byte) (string, error)
+}
+
+// errGrants marks a send that stopped because what the machines' modules may do could not be issued
+// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
+// announcement that hits it is held and asked again.
+var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " +
+	"sent before its grants is refused there")
+
+// grantsRefused is a grant that failed for some machines and not others: their memberships could not
+// be issued, by machine, and only those machines are held back.
+type grantsRefused struct{ nodes map[string]error }
+
+func (g *grantsRefused) Error() string {
+	names := make([]string, 0, len(g.nodes))
+	for n := range g.nodes {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	return fmt.Sprintf("the memberships of %s could not be issued; the first: %v",
+		strings.Join(names, ", "), g.nodes[names[0]])
+}
+
+// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
+// then the rest** (novox/hq issue 249).
+//
+// A merge gave a module a new state; its bundle reached every machine within a minute, and the
+// machines' permissions on the bus did not include the state until somebody pushed by hand: the code
+// arrived before the right to use it. A module that read its new state on start failed its start; the
+// one that was there retried for two minutes. The memberships were issued after the declarations —
+// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on
+// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
+// runtime still on the old code merely holds a grant it does not use yet.
+//
+// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
+// controller's own right to publish memberships and raise buckets is in that list (the precedent of
+// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
+// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
+// on that machine follows a membership issued after its declaration, as it always has.
+//
+// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
+// be said and passed over — "the machines keep what they derive until the next push" — which reported
+// a rollout done that had delivered code its machines could not run. A membership that failed holds
+// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
+// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
+// tried again.
+//
+// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
+// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
+func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
+	digests := map[string]string{}
+	if len(sending) == 0 {
+		return digests, nil
+	}
+	send := func(s readyNode) error {
+		// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
+		// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
+		body, err := s.declared.Body()
+		if err != nil {
+			return err
+		}
+		digest, err := d.declare(ctx, s, body)
+		if err != nil {
+			return err
+		}
+		digests[s.node] = digest
+		return nil
+	}
+	var rest []readyNode
+	for _, s := range sending {
+		if holder != "" && s.node == holder {
+			if err := send(s); err != nil {
+				return digests, err
+			}
+			continue
+		}
+		rest = append(rest, s)
+	}
+	held := map[string]error{}
+	if err := d.grant(ctx, sending); err != nil {
+		var some *grantsRefused
+		if !errors.As(err, &some) {
+			var names []string
+			for _, s := range rest {
+				names = append(names, s.node)
+			}
+			if len(names) == 0 {
+				return digests, fmt.Errorf("%w: %w", errGrants, err)
+			}
+			return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
+		}
+		held = some.nodes
+	}
+	var notSent []string
+	for _, s := range rest {
+		if _, refused := held[s.node]; refused {
+			notSent = append(notSent, s.node)
+			continue
+		}
+		if err := send(s); err != nil {
+			return digests, err
+		}
+	}
+	if len(notSent) > 0 {
+		return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
+			&grantsRefused{nodes: held})
+	}
+	if len(held) > 0 {
+		// Only the holder's own memberships failed, and it was sent before them.
+		return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
+	}
+	return digests, nil
+}
+
+// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed.
+type overTheBus struct {
+	open   *stores
+	server *link.Server
+	signer link.Signer
+	// indent is put before each "sent" line, for the callers whose output is nested.
+	indent string
+}
+
+func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
+	return issueMemberships(ctx, b.open, b.server, sending)
+}
+
+func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
+	if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
+		return "", err
+	}
+	// After it is away, not before. A digest recorded for something that failed to send would make
+	// the machine look current for a declaration it never received.
+	digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
+	if err != nil {
+		return "", err
+	}
+	if s.declared.BusUsers != "" {
+		// And the user list it carried, so the next send reads whether it must go first from the
+		// list alone (novox/hq issue 249). On the same outliving context as the send's record.
+		kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+		err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
+		cancel()
+		if err != nil {
+			return "", err
+		}
+	}
+	fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
+	return digest, nil
+}
+
+// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
+// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
+// module's new permissions are refused by the bus until that list says them. Among the machines it
+// is moved to the front; not among them, it is added only when it is behind.
+func brokerFirst(names []string, holder string, behind bool) []string {
+	if holder == "" {
+		return names
+	}
+	present := false
+	rest := make([]string, 0, len(names))
+	for _, n := range names {
+		if n == holder {
+			present = true
+			continue
+		}
+		rest = append(rest, n)
+	}
+	if !present && !behind {
+		return names
+	}
+	return append([]string{holder}, rest...)
+}
+
+// brokerBehind is the machine holding the bus — the one whose declaration carries the user list —
+// and, when it is not among the machines named, whether the user list it would be sent now differs
+// from the one it was last sent (novox/hq issue 249).
+//
+// **The user list alone, not the whole declaration.** Read from the whole declaration, any change
+// pending on that machine — an upgrade its policy records rather than rolls out — went with every
+// send anywhere, and a module running there always put it in its first wave. A digest of the list
+// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
+func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
+	inv := open.inventory
+	holders, err := seatHolders(ctx, inv)
+	if err != nil {
+		return "", false, err
+	}
+	h, held := holders[theBrokerSeat]
+	if !held || h.Node == "" {
+		return "", false, nil
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return "", false, err
+	}
+	if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
+		// A holder that is sent no user list carries no grant: nothing to send first.
+		return "", false, nil
+	}
+	for _, n := range names {
+		if n == h.Node {
+			return h.Node, false, nil
+		}
+	}
+	plan, _, err := planFor(ctx, open, h.Node)
+	if err != nil {
+		// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
+		return h.Node, false, nil
+	}
+	list, _, err := busUserList(ctx, inv, plan.Modules)
+	if err != nil {
+		return h.Node, false, nil
+	}
+	sent, err := inv.SentBusUsers(ctx, h.Node)
+	if err != nil {
+		return "", false, err
+	}
+	return h.Node, userListBehind(list, sent), nil
+}
+
+// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
+// empty list composed is never behind: there is nothing for it to carry.
+func userListBehind(now, sentDigest string) bool {
+	return now != "" && digestOf([]byte(now)) != sentDigest
+}
+
 // couldNotBeResolved is what a push ends with when some machines could not be worked out.
 //
 // **After the rest have been sent, never instead of sending them.** It is still an error, because
@@ -658,23 +862,36 @@ func couldNotBeResolved(refusals []string, sent int) error {
 // consumer and refused on the provider would leave one end holding a credential the other has
 // never heard of — which is the state this whole mechanism exists to make impossible.
 func sendTo(ctx context.Context, open *stores, names []string) error {
+	_, err := sendToEach(ctx, open, names)
+	return err
+}
+
+// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine
+// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for
+// the machines it sent waits for that one too.
+func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
 	inv := open.inventory
 	ident, err := openIdentity(ctx)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer ident.Close()
 
 	gens, err := generators(ctx, open)
 	if err != nil {
-		return err
+		return nil, err
 	}
+	holder, behind, err := brokerBehind(ctx, open, names)
+	if err != nil {
+		return nil, err
+	}
+	names = brokerFirst(names, holder, behind)
 
 	// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
 	// converge, which flips the node and then sends it — is not made to wait on itself.
 	ctx, release, err := holdNodes(ctx, open, names)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer release()
 
@@ -703,33 +920,29 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
 		sending = append(sending, readyNode{name, declared})
 	}
 	if len(refusals) > 0 {
-		return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
+		return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
 			len(refusals), strings.Join(refusals, "\n\n"))
 	}
 
 	server, err := connectLink(ctx, nil, nil, nil)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer server.Close()
 
-	for _, s := range sending {
-		body, err := s.declared.Body()
-		if err != nil {
-			return err
-		}
-		if err := link.Declare(ctx, server.Bus(), ident, s.node, body, 15*time.Second); err != nil {
-			return err
-		}
-		if _, err := recordSent(ctx, inv, s.node, body); err != nil {
-			return err
-		}
-		fmt.Printf("  sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
-	}
 	// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
 	// same records the bus's accounts are, so what a runtime serves and what its account may are one
-	// composition. Issued after the declaration, because the runtime it is for arrives with it.
+	// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
-	return issueMemberships(ctx, open, server, sending)
+	// membership is for arrives with the declaration, and a membership waits for it on the bus; the
+	// code arriving first was refused its own state until somebody pushed.
+	if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: "  "}, holder, sending); err != nil {
+		return nil, err
+	}
+	sent := make([]string, 0, len(sending))
+	for _, s := range sending {
+		sent = append(sent, s.node)
+	}
+	return sent, nil
 }
 
 // issueMemberships publishes the membership of every module on the machines just sent.
@@ -747,11 +960,26 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 	if !ok {
 		return nil
 	}
-	// The declarations are sent and recorded by now; a membership that cannot be issued is said
+	// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
-	// and does not unsay them. Every runtime without one serves the shape it derives (ADR 0160), so
+	// raise at start asserts them too, but a module registered and assigned since would otherwise have
-	// the push stands, the first failure is named once, and the next push tries again.
+	// its bucket only after the control plane next restarts — found the first time a module declared
-	issued, failed := 0, 0
+	// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
-	var first error
+	//
+	// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
+	// because the declarations were already away; they are sent after this now — all but the bus's
+	// own machine, sent before it (deliver) — and a module whose state does not exist is a module
+	// that fails its start, so they are not sent and the caller tries again rather than reporting the
+	// rollout done.
+	buckets, err := open.inventory.DeclaredBuckets(ctx)
+	if err != nil {
+		return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
+	}
+	if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
+		return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
+	}
+	// Every membership is tried, and the first failure named once.
+	issued := 0
+	refused := map[string]error{}
 	for _, s := range sent {
 		node := s.node
 		for _, d := range records.Assigned[node] {
@@ -772,10 +1000,9 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 				return err
 			}
 			if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
-				if first == nil {
+				if refused[node] == nil {
-					first = err
+					refused[node] = fmt.Errorf("%s: %w", d.Module, err)
 				}
-				failed++
 				continue
 			}
 			issued++
@@ -784,9 +1011,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 	if issued > 0 {
 		fmt.Printf("  issued %d membership(s)\n", issued)
 	}
-	if failed > 0 {
+	if len(refused) > 0 {
-		fmt.Printf("  %d membership(s) could not be issued; the first: %v — the machines keep what "+
+		// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
-			"they derive until the next push\n", failed, first)
+		// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
+		return &grantsRefused{nodes: refused}
 	}
 	return nil
 }
@@ -898,6 +1126,26 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
 	if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
 		return err
 	}
+	// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
+	// whether it was cancelled the moment it took it.
+	if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
+		return err
+	}
+	// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
+	// registration, so a module reading one may watch it before its owner runs anywhere. One that
+	// nothing declares any more is said and kept — what it holds is data.
+	buckets, err := inv.DeclaredBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	undeclared, err := broker.RaiseBuckets(js, buckets)
+	if err != nil {
+		return err
+	}
+	if len(undeclared) > 0 {
+		fmt.Printf("the bus holds state nothing declares any more, kept because it is data: %s — "+
+			"removing it is a person's act\n", strings.Join(undeclared, ", "))
+	}
 	// And how every module hears what it consumes. Derived from the same records the user list is
 	// composed from, so a module the mesh grants a consumer's subjects has that consumer waiting.
 	// Done on every raise, not only when a credential is issued: every module moved onto this bus
@@ -911,18 +1159,14 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
 		return err
 	}
 	hearing := 0
-	for _, p := range users {
+	for _, c := range broker.ConsumersOf(users) {
-		consumer, needed := broker.ConsumerFor(p)
+		if err := js.EnsureConsumer(c.Consumer); err != nil {
-		if !needed {
+			return fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)
-			continue
-		}
-		if err := js.EnsureConsumer(consumer); err != nil {
-			return fmt.Errorf("how %s on %s hears what it consumes: %w", p.Module, p.Node, err)
 		}
 		hearing++
 	}
-	fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, and %d module(s) "+
+	fmt.Printf("the bus at %s has its streams, %d machine(s) can hear a declaration, %d module(s) "+
-		"can hear what they consume\n", broker.BareAddress(address), len(names), hearing)
+		"can hear what they consume, and %d bucket(s) of state\n", broker.BareAddress(address), len(names), hearing, len(buckets))
 	return nil
 }
 
@@ -984,7 +1228,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
 // never wrote it down: status read "applied, current" over a machine that had just been sent
 // something else. What was sent was sent; the record of it must not depend on the sender living
 // another second. Bounded, so a store that is away does not hold a dying process open for ever.
-func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
+//
+// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
+// what tells a machine held back by a policy or a plan from one a push left behind.
+func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
+	builds map[string]string) (string, error) {
 	kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer cancel()
 	record, err := inv.NodeByName(kept, node)
@@ -992,8 +1240,29 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
 		return "", err
 	}
 	digest := digestOf(body)
-	if err := inv.RecordSent(kept, record.ID, digest); err != nil {
+	if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
 		return "", err
 	}
 	return digest, nil
 }
+
+// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
+// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
+// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
+// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing.
+func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) {
+	for _, node := range asked {
+		lines := unheld[node]
+		if len(lines) == 0 {
+			continue
+		}
+		if named {
+			fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines))
+			for _, u := range lines {
+				fmt.Fprintf(w, "  %s\n", u)
+			}
+			continue
+		}
+		fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
+	}
+}

cmd/mesh-controller/queue.go

@@ -0,0 +1,709 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The build queue, controlled by hand (novox/hq ADR 0219).
+//
+// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
+// may be and never settled — and changed through the controller: an ask cancelled, the queue
+// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
+// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
+//
+// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
+// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
+// rather than waiting for ever on an answer nobody will give.
+
+// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
+// than the holder's worst case (its two passes removing containers and its wait between, 50s).
+const (
+	holderAsks = 15 * time.Second
+	killAnswer = 75 * time.Second
+)
+
+// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
+func dialTheBus() (*broker.JetStream, error) {
+	address, err := broker.BusAddress()
+	if err != nil {
+		return nil, err
+	}
+	js, err := broker.Dial(address)
+	if err != nil {
+		return nil, fmt.Errorf("cannot reach the bus: %w", err)
+	}
+	return js, nil
+}
+
+// queueCommand prints every ask in the build seat's work queue.
+func queueCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("queue", flag.ContinueOnError)
+	asJSON := set.Bool("json", false, "the queue as JSON")
+	if _, err := parseAround(set, args); err != nil {
+		return err
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return err
+	}
+	if *asJSON {
+		body, err := json.MarshalIndent(q, "", "  ")
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(body))
+		return nil
+	}
+	fmt.Print(queueText(q, time.Now()))
+	return nil
+}
+
+// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
+// Never what an ask carries as `held` — that is every artifact the mesh has built.
+func queueText(q link.Queue, now time.Time) string {
+	var b strings.Builder
+	waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
+	fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
+	ago := func(t time.Time) string {
+		if t.IsZero() {
+			return "asked at an unknown time"
+		}
+		return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
+	}
+	what := func(a link.QueuedAsk) string {
+		s := a.Repository
+		if a.Path != "" {
+			s += " at " + a.Path
+		}
+		if a.Ref != "" {
+			s += " on " + a.Ref
+		}
+		return s
+	}
+	if len(running) > 0 {
+		fmt.Fprintln(&b, "\nin flight:")
+		for _, a := range running {
+			where := "taken, not yet said where"
+			switch {
+			case a.On != "":
+				where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
+			case a.Was != "":
+				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
+			}
+			fmt.Fprintf(&b, "  %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
+		}
+	}
+	if len(waiting) > 0 {
+		fmt.Fprintln(&b, "\nwaiting:")
+		for _, a := range waiting {
+			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+		}
+	}
+	if len(dead) > 0 {
+		fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
+		for _, a := range dead {
+			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+		}
+	}
+	switch {
+	case len(q.Asks) == 0:
+		fmt.Fprintln(&b, "nothing is asked of it")
+	default:
+		fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
+	}
+	return b.String()
+}
+
+// cancelCommand drops one waiting or dead ask.
+func cancelCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("cancel <build id>")
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	seat := buildSeatHeld(ctx)
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return err
+	}
+	ask, found := q.Find(args[0])
+	if !found {
+		return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
+			"what came of it", args[0], seat)
+	}
+	said, err := cancelAsk(ctx, js, open, seat, ask)
+	if err != nil {
+		return err
+	}
+	fmt.Println(said)
+	return nil
+}
+
+// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
+//
+// **In this order, and each step for a reason** (novox/hq ADR 0219):
+//  1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
+//     is running — that is `kill`, on the machine running it;
+//  2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
+//  3. for a waiting ask, the worker is read again: one handed out since the queue was read was
+//     taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
+//     read the mark first and ends it as cancelled, or is building it;
+//  4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
+//     handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
+//     at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
+//     that took it before the mark is building it, and only `kill` ends that;
+//  5. the message is deleted by its sequence;
+//  6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
+func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
+	if ask.State == link.AskInFlight && ask.On != "" {
+		return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
+			"ends the build where it runs", ask.ID, ask.On, ask.ID)
+	}
+	if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
+		return "", err
+	}
+	withdraw := func() {
+		if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
+			fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
+		}
+	}
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	switch ask.State {
+	case link.AskWaiting:
+		if taken, err := takenSince(js, worker, ask.Seq); err != nil {
+			withdraw()
+			return "", err
+		} else if taken {
+			withdraw()
+			return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
+				"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
+				"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
+		}
+	case link.AskInFlight:
+		if started, err := startedSince(ctx, js, seat, ask); err != nil {
+			withdraw()
+			return "", err
+		} else if started != "" {
+			withdraw()
+			return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
+				"ends it there", ask.ID, started, ask.ID)
+		}
+	}
+	if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
+		!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
+		return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
+			"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
+	}
+	recordCancelled(ctx, open, ask)
+	return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
+		"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
+}
+
+// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
+// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
+var holderLooks = 5 * time.Second
+
+// startedSince waits out a holder's look at the cancelled set and says the machine that started the
+// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
+// a start of a build.
+func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
+	select {
+	case <-ctx.Done():
+		return "", ctx.Err()
+	case <-time.After(holderLooks):
+	}
+	since := time.Now().Add(-7 * 24 * time.Hour)
+	if !ask.AskedAt.IsZero() {
+		since = ask.AskedAt.Add(-time.Minute)
+	}
+	heard, err := link.ReadBuildEvents(ctx, js, seat, since)
+	if err != nil {
+		return "", err
+	}
+	s, ok := heard.Started[ask.ID]
+	if !ok || heard.Outcomes[ask.ID] {
+		return "", nil
+	}
+	at, _ := time.Parse(time.RFC3339Nano, s.At)
+	if ask.Started.IsZero() || at.After(ask.Started) {
+		return s.On, nil
+	}
+	return "", nil
+}
+
+// takenSince is whether the worker has handed out the ask at this sequence.
+func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
+	info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
+	if errors.Is(err, nats.ErrConsumerNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
+	}
+	return info.Delivered.Stream >= seq, nil
+}
+
+// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
+// then the plan that asked for it — by the id it asked with, or by repository and path.
+func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
+	r := ask.Request
+	result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
+		Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
+	_ = builds{open.inventory, open}.Built(ctx, result)
+}
+
+// clearCommand cancels every waiting ask, and with --dead every dead one too.
+func clearCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("clear", flag.ContinueOnError)
+	dead := set.Bool("dead", false, "the dead asks too")
+	if _, err := parseAround(set, args); err != nil {
+		return err
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	seat := buildSeatHeld(ctx)
+	said, err := clearQueue(ctx, js, open, seat, *dead)
+	fmt.Print(said)
+	return err
+}
+
+// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
+func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return "", err
+	}
+	var b strings.Builder
+	cancelled, refused := 0, 0
+	for _, a := range q.Asks {
+		if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
+			continue
+		}
+		said, err := cancelAsk(ctx, js, open, seat, a)
+		if err != nil {
+			refused++
+			fmt.Fprintf(&b, "  %s: %v\n", a.ID, err)
+			continue
+		}
+		cancelled++
+		fmt.Fprintf(&b, "  %s\n", said)
+	}
+	what := "waiting"
+	if dead {
+		what = "waiting and dead"
+	}
+	fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
+	if refused > 0 {
+		fmt.Fprintf(&b, ", %d could not be", refused)
+	}
+	fmt.Fprintln(&b)
+	if n := len(q.Of(link.AskInFlight)); n > 0 {
+		fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
+	}
+	if n := len(q.Of(link.AskDead)); n > 0 && !dead {
+		fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
+	}
+	if refused > 0 {
+		return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
+	}
+	return b.String(), nil
+}
+
+// rebuildCommand asks the module's current source again — or, given a build's id, that build's
+// repository, path and ref — under a new id.
+func rebuildCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("rebuild <module | build id>")
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	var source buildSource
+	var path, ref, module string
+	if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
+		return err
+	} else if found {
+		source, module = sourceOfBuild(b, entries)
+		path, ref = b.Path, b.Ref
+		// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
+		// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
+		// that commit out over everything since. Building that commit again is `replay`, which says
+		// what it would do and refuses to register it while anything newer is asked or registered.
+		if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
+			ref = followedBranch(e.Source.Ref)
+			follows := ref
+			if follows == "" {
+				follows = "the repository's default branch"
+			}
+			fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
+				"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
+		}
+	} else if e, known := entryNamed(entries, args[0]); known {
+		// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
+		source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
+		path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
+	} else {
+		return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
+	}
+
+	// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
+	// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
+	if module != "" {
+		joined, said, err := joinAPlan(ctx, open, module)
+		if err != nil {
+			return err
+		}
+		if joined {
+			fmt.Println(said)
+			return nil
+		}
+	}
+	id, err := askABuild(ctx, source, path, ref)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("rebuild asked as %s\n", id)
+	return nil
+}
+
+// entryNamed is the catalogue's entry for a module.
+func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
+	for _, e := range entries {
+		if e.Manifest.Module == name {
+			return e, true
+		}
+	}
+	return inventory.Entry{}, false
+}
+
+// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
+// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
+// (ADR 0111) — else the repository as it was cloned.
+func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
+	for _, e := range entries {
+		if (b.Module != "" && e.Manifest.Module == b.Module) ||
+			(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
+			return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
+		}
+	}
+	return buildSource{Repository: b.Repository}, b.Module
+}
+
+// replayCommand asks a recorded build's repository and path again at the commit it built.
+func replayCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("replay", flag.ContinueOnError)
+	register := set.Bool("register", false, "register what it builds, as any build is")
+	older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) != 1 {
+		return errors.New("replay <build id> [--register [--older]]")
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	b, found, err := inv.BuildByID(ctx, positionals[0])
+	if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("no build %s is recorded", positionals[0])
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	source, module := sourceOfBuild(b, entries)
+	var history []inventory.Build
+	if module != "" {
+		if history, err = inv.Builds(ctx, module, 100); err != nil {
+			return err
+		}
+	}
+	// What is asked of the module and not yet answered, when the replay would be registered.
+	var outstanding []string
+	if *register {
+		js, err := dialTheBus()
+		if err != nil {
+			return err
+		}
+		q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
+		js.Close()
+		if err != nil {
+			return err
+		}
+		plans, err := inv.OpenPlans(ctx)
+		if err != nil {
+			return err
+		}
+		outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
+	}
+	if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
+		return err
+	}
+	id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
+	if err != nil {
+		return err
+	}
+	fmt.Println(replaySaid(b, module, id, *register))
+	return nil
+}
+
+// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
+//
+// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
+// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
+// older commit is newer than everything since, and would roll that older commit out as the module's
+// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
+// --register says otherwise, and --register is refused when a newer build of a different commit is
+// registered, unless --older says that is the point.
+//
+// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
+// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
+// and their builds — made from newer source — would be recorded and never registered (issue 219
+// orders by ask), the plan waiting on them sending the older commit instead.
+func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
+	outstanding []string) error {
+	if b.Commit == "" {
+		return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
+			"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
+	}
+	if older && !register {
+		return errors.New("--older only says what --register may do; a dry run registers nothing")
+	}
+	if register && len(outstanding) > 0 {
+		return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
+			"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
+			orNone(module), strings.Join(outstanding, "; "), b.ID)
+	}
+	if !register || older {
+		return nil
+	}
+	for _, h := range history {
+		if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
+			continue
+		}
+		if h.AskedOrAt().After(b.AskedOrAt()) {
+			return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
+				"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
+				"without --register looks at it; --register --older registers it anyway",
+				module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
+		}
+	}
+	return nil
+}
+
+// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
+func replaySaid(b inventory.Build, module, id string, register bool) string {
+	what := b.Repository
+	if module != "" {
+		what = module
+	}
+	said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
+	if !register {
+		return said + "\n  a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
+			"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
+	}
+	return said + "\n  registered when it is built, as the module's current version — newer than every build " +
+		"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
+}
+
+// killCommand finds the machine running a build and asks its holder to end it.
+func killCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("kill <build id>")
+	}
+	id := args[0]
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	since := time.Now().Add(-7 * 24 * time.Hour)
+	if at, ok := link.BuildAskedAt(id); ok {
+		since = at.Add(-time.Minute)
+	}
+	heard, err := link.ReadBuildEvents(ctx, js, seat, since)
+	if err != nil {
+		return err
+	}
+	started, ok := heard.Started[id]
+	if !ok {
+		return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
+	}
+	if heard.Outcomes[id] {
+		return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
+	}
+	answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
+	if err != nil {
+		return err
+	}
+	return printHolderAnswer(started.On, answer)
+}
+
+// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
+func pauseCommand(ctx context.Context, verb string, args []string) error {
+	if len(args) > 1 {
+		return fmt.Errorf("%s [node]", verb)
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	nodes := args
+	if len(nodes) == 0 {
+		if nodes, err = buildSeatHolders(ctx, seat); err != nil {
+			return err
+		}
+		if len(nodes) == 0 {
+			return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
+		}
+	}
+	failed := 0
+	for _, node := range nodes {
+		answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
+		if err != nil {
+			fmt.Printf("%s: %v\n", node, err)
+			failed++
+			continue
+		}
+		if err := printHolderAnswer(node, answer); err != nil {
+			failed++
+		}
+	}
+	if failed > 0 {
+		return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
+	}
+	return nil
+}
+
+// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
+func printHolderAnswer(node string, answer link.Answer) error {
+	if answer.Error != "" {
+		fmt.Printf("%s: %s\n", node, answer.Error)
+		return errors.New(answer.Error)
+	}
+	var said struct {
+		Said string `json:"said"`
+	}
+	if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
+		fmt.Printf("%s: %s\n", node, said.Said)
+		return nil
+	}
+	fmt.Printf("%s: %s\n", node, string(answer.Result))
+	return nil
+}
+
+// buildSeatHolders is every machine an assigned module holding the build seat runs on.
+func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
+	open, err := openStores(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer open.Close()
+	entries, err := open.inventory.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return holdersAmong(entries, seat), nil
+}
+
+// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
+func holdersAmong(entries []inventory.Entry, seat string) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, e := range entries {
+		if !e.Manifest.ClaimsSeat(seat) {
+			continue
+		}
+		for _, n := range e.On {
+			if !seen[n] {
+				seen[n] = true
+				out = append(out, n)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
+// by repository and path, and every open plan holding it not yet built.
+func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
+	var out []string
+	for _, a := range q.Asks {
+		if repositoryMatches(a.Repository, repository) && a.Path == path {
+			out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
+		}
+	}
+	if module == "" {
+		return out
+	}
+	for _, p := range plans {
+		if !p.Open() {
+			continue
+		}
+		for _, tier := range p.Tiers {
+			for _, m := range tier {
+				if m == module {
+					if st := p.Modules[m]; st == nil || st.State != "built" {
+						out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
+					}
+				}
+			}
+		}
+	}
+	return out
+}

cmd/mesh-controller/queue_test.go

@@ -0,0 +1,772 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
+//
+//	docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
+//	make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
+//	MESH_TEST_NATS=nats://127.0.0.1:14294 \
+//	MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
+//	  go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
+
+// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
+func asksRecorded(t *testing.T) *[]string {
+	t.Helper()
+	var asked []string
+	was := askABuild
+	askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
+		id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
+		asked = append(asked, source.Repository+"#"+id)
+		return id, nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	return &asked
+}
+
+// twoTiers registers two modules, b standing on a, each with a source a plan asks.
+func twoTiers(t *testing.T, open *stores) {
+	t.Helper()
+	for _, name := range []string{"a", "b"} {
+		if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
+			inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// A failed plan is retried: its failed module asked again under a new id, the plan building at that
+// tier, and when that build comes in the plan goes on and asks its next tier.
+func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	before := time.Now().UTC().Add(-time.Hour)
+	failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
+		Note: "a failed to build in tier 0",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
+			Why: link.KilledByHand}}}
+	if err := open.inventory.SavePlan(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+
+	said, err := retryPlan(ctx, open, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, err := open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := p.Modules["a"]
+	if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
+		t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
+	}
+	if !strings.Contains(said, a.Build) {
+		t.Errorf("retry does not say the new id: %q", said)
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+
+	// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
+	planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
+	if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
+		t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
+	}
+	asking, _ := link.BuildAskedAt(a.Build)
+	planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
+	p, err = open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
+		t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
+	}
+	if len(*asked) != 2 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// What retry refuses, and says why.
+func TestRetryRefusesWhatItCannotResume(t *testing.T) {
+	at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	plan := func(id, state string, created time.Time) inventory.Plan {
+		return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
+			Created: created, State: state, Tiers: [][]string{{"a"}},
+			Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
+	}
+	failed := plan("plan-1", inventory.PlanFailed, at)
+	for _, c := range []struct {
+		p      inventory.Plan
+		others []inventory.Plan
+		says   string
+	}{
+		{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
+		{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
+		{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
+		{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
+	} {
+		err := retryRefusal(c.p, c.others)
+		if err == nil || !strings.Contains(err.Error(), c.says) {
+			t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
+		}
+	}
+	stopped := failed
+	stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
+	stopped.Note = "a stopped at its first machine"
+	if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
+		t.Errorf("a plan with nothing failed to build was retried: %v", err)
+	}
+	// Another branch's newer plan, an older one, and a failed one do not supersede it.
+	other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
+	other.Branch = "release"
+	if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
+		plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
+		t.Errorf("refused for a plan that does not supersede it: %v", err)
+	}
+}
+
+// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
+func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	before := time.Now().UTC().Add(-time.Hour)
+	failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
+		Note:    "a failed to build in tier 0",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
+	if err := open.inventory.SavePlan(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"a"}); err != nil {
+		t.Fatal(err)
+	}
+	p, err := open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
+		t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+	// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
+	if err := rebuildCommand(ctx, []string{"b"}); err != nil {
+		t.Fatal(err)
+	}
+	if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
+		t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
+	}
+	if len(*asked) != 2 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
+func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
+	at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
+		Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
+	newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
+		State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
+	if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
+		t.Fatal("joined a failed plan a newer open one supersedes")
+	}
+	if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
+		t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
+	}
+	built := failed
+	built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
+	if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
+		t.Fatal("joined a plan that has the module built")
+	}
+}
+
+// replay is a dry run unless registered, and registering an older commit than one registered since
+// is refused unless --older says it is meant (novox/hq issue 207).
+func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
+	asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
+	newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
+	history := []inventory.Build{newer, old}
+
+	if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
+		t.Errorf("a dry run was refused: %v", err)
+	}
+	err := replayRefusal(old, "a", history, true, false, nil)
+	if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
+		t.Errorf("registering an older commit than the one registered was not refused: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
+		t.Errorf("--register --older was refused: %v", err)
+	}
+	if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
+		t.Errorf("registering the newest build again was refused: %v", err)
+	}
+	// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
+	same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
+	broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
+	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
+		t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
+	}
+	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
+		t.Error("a build that recorded no commit was replayed")
+	}
+	if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
+		t.Error("--older without --register was taken")
+	}
+	// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
+	// holding it unbuilt, would be replaced by a replay asked now (issue 219).
+	q := link.Queue{Asks: []link.QueuedAsk{
+		{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
+		{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
+		{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
+	}}
+	plans := []inventory.Plan{
+		{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
+		{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
+		{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
+	}
+	outstanding := outstandingFor("a", "novox/a", "", q, plans)
+	if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
+		t.Fatalf("outstanding: %v", outstanding)
+	}
+	if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
+		t.Errorf("registered over an outstanding ask: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
+		t.Errorf("a dry run was refused for what is outstanding: %v", err)
+	}
+	if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
+		t.Errorf("a dry replay does not say what it is: %q", said)
+	}
+	if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
+		t.Errorf("a registered replay does not say what it does: %q", said)
+	}
+}
+
+// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
+// paused on some holders only is not a reason the plan is waiting.
+func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
+	now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	asked := now.Add(-12 * time.Minute)
+	p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
+		Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
+
+	all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
+	line := planLineWith(p, now, all)
+	if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan on a paused seat reads %q", line)
+	}
+	if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
+		t.Errorf("status --json says %+v", st)
+	}
+	if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
+		t.Errorf("a plan waiting on a paused seat counted late")
+	}
+
+	longAgo := now.Add(-25 * time.Hour)
+	forgotten := p
+	forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
+	if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan paused over a day reads %q", line)
+	}
+
+	some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
+	if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
+		t.Fatalf("%+v", some)
+	}
+	if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
+		t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
+	}
+	if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
+		t.Errorf("status --json: %+v", st)
+	}
+	// A plan rolling out, or with nothing asked, is not waiting on the seat.
+	rolling := p
+	rolling.State = inventory.PlanRolling
+	if _, paused := pausedWaiting(rolling, all, now); paused {
+		t.Error("a rolling plan reads as waiting on the build seat")
+	}
+}
+
+// The queue's verbs are the controller seat's, each to the command it names.
+func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
+	for _, c := range []struct {
+		verb string
+		args map[string]any
+		want []string
+	}{
+		{"queue", nil, []string{"queue"}},
+		{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
+		{"clear", nil, []string{"clear"}},
+		{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
+		{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
+		{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
+		{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
+		{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
+		{"pause", nil, []string{"pause"}},
+		{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
+		{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
+	} {
+		got, err := argvFor(c.verb, c.args)
+		if err != nil || !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
+		}
+	}
+	if _, err := argvFor("cancel", nil); err == nil {
+		t.Error("cancel without an id was taken")
+	}
+	declared := map[string]bool{}
+	for _, v := range catalogue.ControllerVerbs {
+		declared[v.Name] = true
+	}
+	for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
+		if !declared[v] {
+			t.Errorf("%s is not a verb of the controller seat", v)
+		}
+	}
+}
+
+// --- against a real bus -----------------------------------------------------------------------
+
+// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
+// pointed at it, and a function that asks the seat one build.
+func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
+	t.Helper()
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	t.Setenv(broker.NATSVar, url)
+	js, err := broker.Dial(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
+		Emits: []string{"started", "built", "log.*", "paused.*"}}
+	if err := broker.AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
+	if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
+		link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
+		_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
+		_ = js.Context().PurgeStream(broker.EventsStream)
+	})
+	ask := func(id, repository string) link.BuildRequest {
+		r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
+		body, _ := json.Marshal(r)
+		if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
+			t.Fatal(err)
+		}
+		return r
+	}
+	return js, ask
+}
+
+// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
+func deadOne(t *testing.T, js *broker.JetStream) {
+	t.Helper()
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	for i := 0; i < worker.MaxDeliver; i++ {
+		msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
+		if err != nil {
+			t.Fatalf("delivery %d: %v", i+1, err)
+		}
+		_ = msgs[0].Nak()
+	}
+	// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
+	// then, and stops counting it pending.
+	if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
+		t.Fatalf("an ask past its deliveries was delivered again")
+	}
+}
+
+// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
+// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
+// is still found.
+func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	twoTiers(t, open)
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	asked, _ := link.BuildAskedAt(id)
+	plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
+		t.Fatalf("the queue reads %+v", q)
+	}
+	if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
+		strings.Contains(text, "secret-ish") {
+		t.Errorf("the queue says:\n%s", text)
+	}
+
+	if err := cancelCommand(ctx, []string{id}); err != nil {
+		t.Fatal(err)
+	}
+	if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
+		t.Fatalf("the ask is still queued: %+v", q.Asks)
+	}
+	if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
+		t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
+	}
+	b, found, err := open.inventory.BuildByID(ctx, id)
+	if err != nil || !found || b.Failed != link.CancelledByHand {
+		t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
+	}
+	p, err := open.inventory.PlanByID(ctx, plan.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
+		t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
+	}
+	if err := cancelCommand(ctx, []string{id}); err == nil {
+		t.Error("an ask cancelled already was cancelled again")
+	}
+}
+
+// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
+func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	start := time.Now()
+	dead := link.NewBuildID(start)
+	ask(dead, "https://forge.example/novox/dead.git")
+	deadOne(t, js)
+	var waiting []string
+	for i := 1; i <= 2; i++ {
+		id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
+		waiting = append(waiting, id)
+		ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
+	}
+
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
+		t.Fatalf("the queue reads %+v", q)
+	}
+	said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
+		t.Errorf("clear said:\n%s", said)
+	}
+	q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
+		t.Fatalf("after clear the queue is %+v", q.Asks)
+	}
+	if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
+		t.Fatal(err)
+	}
+	if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
+		t.Fatalf("clear --dead left %+v", q.Asks)
+	}
+	for _, id := range append(waiting, dead) {
+		if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
+			t.Errorf("%s is recorded as %+v", id, b)
+		}
+	}
+}
+
+// An ask in flight is not cancelled: kill ends it where it runs.
+func TestCancelRefusesAnAskInFlight(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+	if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "ace" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
+	if err == nil || !strings.Contains(err.Error(), "kill "+id) {
+		t.Fatalf("an ask in flight was cancelled: %v", err)
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a refused cancel recorded an outcome")
+	}
+}
+
+// kill finds the machine from the build's start and asks that machine's holder; pause asks the
+// machine named. Each prints what the holder answered, and a refusal is the command's failure.
+func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
+	js, _ := aBuildQueue(t)
+	ctx := t.Context()
+	asked := map[string]string{}
+	handlers := map[string]link.ToolHandler{
+		"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
+			var args struct{ ID string }
+			_ = json.Unmarshal(raw, &args)
+			asked["kill"] = args.ID
+			if args.ID != "build-running" {
+				return nil, fmt.Errorf("ace is not building %s", args.ID)
+			}
+			return map[string]any{"said": "killed " + args.ID}, nil
+		},
+		"pause": func(context.Context, json.RawMessage) (any, error) {
+			asked["pause"] = "ace"
+			return map[string]any{"said": "ace is paused"}, nil
+		},
+	}
+	stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stop()
+	for _, id := range []string{"build-running", "build-other"} {
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := killCommand(ctx, []string{"build-running"}); err != nil {
+		t.Fatal(err)
+	}
+	if asked["kill"] != "build-running" {
+		t.Fatalf("the holder was asked %v", asked)
+	}
+	if err := killCommand(ctx, []string{"build-other"}); err == nil {
+		t.Error("the holder's refusal was not the command's")
+	}
+	if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
+		t.Errorf("a build nobody started: %v", err)
+	}
+	if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
+		t.Fatalf("pause: %v %v", err, asked)
+	}
+	if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
+		t.Error("a machine nothing answers on was resumed")
+	}
+}
+
+// A plan that stopped at its first machine is retried: the module sent to that machine again, the
+// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
+func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var sentTo [][]string
+	was := sendRollout
+	sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
+		sentTo = append(sentTo, names)
+		return names, nil
+	}
+	t.Cleanup(func() { sendRollout = was })
+
+	long := time.Now().UTC().Add(-2 * time.Hour)
+	stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
+		Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
+			First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
+	if err := open.inventory.SavePlan(ctx, stopped); err != nil {
+		t.Fatal(err)
+	}
+
+	// A newer plan holding a refuses it: sending the older build would put it back.
+	newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
+		State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
+	if err := open.inventory.SavePlan(ctx, newer); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
+		t.Fatalf("retried under a newer plan: %v", err)
+	}
+	newer.State = inventory.PlanSuperseded
+	if err := open.inventory.SavePlan(ctx, newer); err != nil {
+		t.Fatal(err)
+	}
+
+	said, err := retryPlan(ctx, open, stopped.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
+		t.Fatalf("sent %v; said %q", sentTo, said)
+	}
+	p, err := open.inventory.PlanByID(ctx, stopped.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := p.Modules["a"]
+	if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
+		t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
+	}
+	// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
+	advancePlans(ctx, open)
+	if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
+		t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
+// the plan, asked later, never answers it (novox/hq ADR 0219).
+func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := time.Now().UTC().Add(-time.Minute)
+	plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+	planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
+	p, _ := open.inventory.PlanByID(ctx, plan.ID)
+	if p.Modules["a"].State != "asked" {
+		t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
+	}
+	// From the records too: a later build of the module recorded is not the plan's.
+	recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
+	if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
+		t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
+	}
+	planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
+	if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
+		t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
+	}
+}
+
+// rebuild of a build made at a commit asks what the module follows now, never the commit.
+func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var refs []string
+	was := askABuild
+	askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
+		refs = append(refs, ref)
+		return was(c, source, path, ref)
+	}
+	if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
+		Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
+		t.Fatalf("asked %v at %v", *asked, refs)
+	}
+}
+
+// An ask the worker counts out that no machine said it started is cancelled only if no start comes
+// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
+func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	was := holderLooks
+	holderLooks = 300 * time.Millisecond
+	t.Cleanup(func() { holderLooks = was })
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	// The holder that took it says it started, while the cancel waits.
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		_, _ = js.Context().Publish(link.BuildStarted(), started)
+	}()
+	if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
+		t.Fatalf("a build that started was cancelled: %v", err)
+	}
+	if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
+		t.Error("the withdrawn cancel is still marked")
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a withdrawn cancel recorded an outcome")
+	}
+}

cmd/mesh-controller/readable.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"sort"
 	"time"
@@ -73,6 +74,15 @@ type meshStatus struct {
 	// alone. A document without this called a machine well while a predecessor's chain refused
 	// what the mesh declared open.
 	Filtered []machineFiltered `json:"filtered,omitempty"`
+	// Unheld is every module on a machine whose resources are applied through a seat nothing on
+	// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
+	// dependency is met. Reported, not refused, until the switch.
+	Unheld []catalogue.Unheld `json:"unheld,omitempty"`
+	// Failing is every consumer a provider says it keeps failing, with the class of error, since
+	// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
+	// document without this called the mesh well while the identity provider refused every consumer
+	// for a day (04-ISSUES/179).
+	Failing []inventory.ProviderStanding `json:"failing,omitempty"`
 }
 
 // machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -171,7 +181,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
 	out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
 		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
 		Reported: []machineReported{}, Unresolved: []machineUnresolved{},
-		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
+		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
 	// In a stated order, so two readings of an unchanged mesh are the same document.
 	untakenNodes := make([]string, 0, len(asked.untaken))
 	for name := range asked.untaken {
@@ -204,6 +214,8 @@ func statusAsJSON(asked answers) ([]byte, error) {
 			out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
 		}
 	}
+	out.Unheld = asked.unheld
+	out.Failing = asked.failing
 	for name := range asked.refused {
 		out.Unresolved = append(out.Unresolved, machineUnresolved{
 			Node: name, Problem: asked.refused[name]})

cmd/mesh-controller/release_plan.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"sort"
@@ -183,6 +184,7 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
 	return inventory.Plan{
 		ID:         fmt.Sprintf("plan-%d", time.Now().UnixNano()),
 		Repository: m.Owner + "/" + m.Repo,
+		Branch:     m.Base,
 		Commit:     m.Commit,
 		Created:    time.Now().UTC(),
 		State:      inventory.PlanBuilding,
@@ -191,6 +193,57 @@ func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inv
 	}
 }
 
+// supersededBy is what a newer plan takes over from the open plans it supersedes (novox/hq issue
+// 254, ADR 0218): the modules they had not finished, and those plans closed as superseded.
+//
+// **A merge looked at no plan but its own.** Two merges of one repository a few minutes apart were
+// two open plans asking for the same modules, each sending machines what it built; and a plan that
+// would never move again — waiting on a report that could not come, at 97b1b2b — stayed open for
+// ever beside the newer ones, read as work in progress by everyone who looked. The newer merge is the
+// newer intent for that repository and branch, so its plan takes over: every open plan of the same
+// repository and branch **created before it** — by the time the plans were made, never by comparing
+// commits, which have no order of their own — gives up the modules it had not built, and those are
+// planned again in the newer plan beside what the newer merge moved.
+//
+// "Not built" is a module not yet asked, or asked and not answered; **and a module built and not
+// yet sent to its machines**, where its policy rolls it out: closed, the older plan would never send
+// it, and the catalogue announces no move for a rebuild (issue 189), so the newer plan builds and
+// sends it. A build the older plan asked still finishes and registers as any build does — ordered by
+// when it was asked (issue 219), so the newer plan's ask, made later, is the one that stands.
+//
+// A plan with no branch recorded is from before branches were kept, and is superseded by the next
+// plan of its repository: what it had not built is folded in, so nothing is lost by it.
+func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(string) bool) ([]string, []inventory.Plan) {
+	folded := map[string]bool{}
+	var closed []inventory.Plan
+	for _, old := range open {
+		if old.ID == newer.ID || !old.Open() || !strings.EqualFold(old.Repository, newer.Repository) ||
+			(old.Branch != "" && old.Branch != newer.Branch) || !old.Created.Before(newer.Created) {
+			continue
+		}
+		var took []string
+		for name, s := range old.Modules {
+			if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) {
+				folded[name] = true
+				took = append(took, name)
+			}
+		}
+		sort.Strings(took)
+		old.State = inventory.PlanSuperseded
+		old.Note = fmt.Sprintf("superseded at tier %d by %s (%s at %s)", old.Tier, newer.ID, newer.Repository, short(newer.Commit))
+		if len(took) > 0 {
+			old.Note += "; " + strings.Join(took, ", ") + " planned there again"
+		}
+		closed = append(closed, old)
+	}
+	out := make([]string, 0, len(folded))
+	for name := range folded {
+		out = append(out, name)
+	}
+	sort.Strings(out)
+	return out, closed
+}
+
 // gates is what the next tier needs running from this one: a module of the tier that a later
 // tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by
 // the machines running it before the next tier is asked. A base an image stands on need only be
@@ -229,6 +282,22 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
 }
 
 // applied says whether every machine running the module has reported since the module was built.
+// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
+// after that machine was sent the build (novox/hq issue 256).
+func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) {
+	at := map[string]*time.Time{}
+	for _, r := range reports {
+		at[r.Node] = r.At
+	}
+	var waiting []string
+	for _, n := range running {
+		if t := at[n]; t == nil || t.Before(since(n)) {
+			waiting = append(waiting, n)
+		}
+	}
+	return len(waiting) == 0, waiting
+}
+
 func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
 	at := map[string]*time.Time{}
 	for _, r := range reports {
@@ -255,37 +324,52 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 	for _, e := range entries {
 		byName[e.Manifest.Module] = e
 	}
-	now := time.Now().UTC()
 	for _, name := range p.Tiers[p.Tier] {
-		state := p.Modules[name]
+		askModule(ctx, p, name, byName)
-		if state == nil {
-			state = &inventory.PlanModule{}
-			p.Modules[name] = state
-		}
-		e, known := byName[name]
-		if !known {
-			state.State = "failed"
-			state.Why = "no longer in the catalogue"
-			p.State = inventory.PlanFailed
-			p.Note = name + " is no longer in the catalogue"
-			continue
-		}
-		source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
-		fmt.Printf("  tier %d: ", p.Tier)
-		// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
-		if err := buildOne(ctx, source, e.Source.Path, followedBranch(e.Source.Ref), 0); err != nil {
-			state.State = "failed"
-			state.Why = err.Error()
-			p.State = inventory.PlanFailed
-			p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
-			continue
-		}
-		state.State = "asked"
-		state.AskedAt = &now
 	}
 	return nil
 }
 
+// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
+// variable so a test of what a plan does around an ask needs no build machine.
+var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
+	return buildOneAsked(ctx, source, path, ref, 0, false)
+}
+
+// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
+// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
+func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
+	now := time.Now().UTC()
+	state := p.Modules[name]
+	if state == nil {
+		state = &inventory.PlanModule{}
+		p.Modules[name] = state
+	}
+	e, known := byName[name]
+	if !known {
+		state.State = "failed"
+		state.Why = "no longer in the catalogue"
+		p.State = inventory.PlanFailed
+		p.Note = name + " is no longer in the catalogue"
+		return
+	}
+	source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
+	fmt.Printf("  tier %d: ", p.Tier)
+	// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
+	id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
+	if err != nil {
+		state.State = "failed"
+		state.Why = err.Error()
+		p.State = inventory.PlanFailed
+		p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
+		return
+	}
+	state.State = "asked"
+	state.AskedAt = &now
+	state.Build = id
+	state.Why, state.Commit, state.BuiltAt = "", "", nil
+}
+
 // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
 // advances what that completes. Called from the daemon's take-in of every outcome.
 //
@@ -294,8 +378,17 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 // the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
 // would send machines, and the next tier, what the later merge replaced. asked is zero when the
 // build's request time is not known, and such an outcome is taken as before.
-func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) {
+func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
 	inv := open.inventory
+	// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
+	// than write over what the holder is about to save. Not taken, it is still in the build records,
+	// which the holder settles the plan from (issue 214).
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
+		return
+	}
+	defer release()
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
 		fmt.Printf("plans: cannot read them: %v\n", err)
@@ -321,7 +414,17 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			state = &inventory.PlanModule{}
 			p.Modules[module] = state
 		}
-		if askedBefore(asked, state.AskedAt) {
+		// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
+		// is, as before, when it was asked at or after the plan's ask (issue 219).
+		// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
+		// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
+		// build, made from other source, and settling the plan with it would send that. A plan from
+		// before ids were kept is matched as it was: by when the build was asked (issue 219).
+		if state.Build != "" {
+			if state.Build != id {
+				continue
+			}
+		} else if askedBefore(asked, state.AskedAt) {
 			continue
 		}
 		if failed != "" {
@@ -329,6 +432,10 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			state.Why = failed
 			p.State = inventory.PlanFailed
 			p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier)
+			sayUnsent(p, func(m string) bool {
+				u, err := inv.UpgradeOf(ctx, m)
+				return err == nil && u.RollOut
+			})
 		} else {
 			state.State = "built"
 			state.BuiltAt = &now
@@ -342,13 +449,30 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
 		}
 	}
-	advancePlans(ctx, open)
+	advanceHeld(ctx, open)
 }
 
 // advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
 // and whose gates are applied gives way to the next; the last tier done is the plan done. Called
 // after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
+//
+// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
+// controllers — the old and the new while a machine hands its controller over — would each ask a
+// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
 func advancePlans(ctx context.Context, open *stores) {
+	release, err := open.inventory.HoldPlans(ctx, false)
+	if err != nil {
+		if !errors.Is(err, inventory.ErrPlansBusy) {
+			fmt.Printf("plans: cannot hold them: %v\n", err)
+		}
+		return
+	}
+	defer release()
+	advanceHeld(ctx, open)
+}
+
+// advanceHeld is advancePlans for a caller already holding the plans.
+func advanceHeld(ctx context.Context, open *stores) {
 	inv := open.inventory
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
@@ -373,8 +497,17 @@ func advancePlans(ctx context.Context, open *stores) {
 			moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
 			if err != nil {
 				fmt.Printf("%s: %v\n", p.ID, err)
+				// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
+				// the state is left as it was and the step is tried again on the next tick.
+				p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
+				if err := inv.SavePlan(ctx, *p); err != nil {
+					fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
+				}
 				break
 			}
+			if p.State == inventory.PlanFailed {
+				sayUnsent(p, rollsOut)
+			}
 			if err := inv.SavePlan(ctx, *p); err != nil {
 				fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
 				break
@@ -415,6 +548,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
 	// outcome, whoever was listening.
 	recorded := map[string][]inventory.Build{}
+	byID := map[string]inventory.Build{}
 	for _, m := range tier {
 		if s := p.Modules[m]; s != nil && s.State == "asked" {
 			builds, err := inv.Builds(ctx, m, 5)
@@ -422,9 +556,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 				return false, err
 			}
 			recorded[m] = builds
+			// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
+			if s.Build != "" {
+				b, found, err := inv.BuildByID(ctx, s.Build)
+				if err != nil {
+					return false, err
+				}
+				if found {
+					byID[s.Build] = b
+				}
+			}
 		}
 	}
-	if settleFromRecords(p, tier, recorded) {
+	if settleFromRecords(p, tier, recorded, byID) {
 		return true, nil
 	}
 	// Asked: wait for every build.
@@ -443,6 +587,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// a module that packages another repository's source, keeps its commit; the catalogue announces
 	// no move for it and its machines would keep the old image until somebody pushed (novox/hq
 	// issue 189). A module whose policy records is built and left, as its policy says.
+	//
+	// **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR
+	// 0218). The plan sent every machine running the module at once, and the operator's policy —
+	// one at a time, stopping at the first that fails, which an announced upgrade honours — was not
+	// read here at all: a module whose new declarations broke it broke everywhere in the same
+	// minute. Now the first machine is sent, the plan records it and waits for that machine's report
+	// after the send to say it applied what it was sent; only then are the rest sent. A first machine
+	// that fails or refuses stops the module's rollout and the plan with it, the rest untouched.
+	var pending []string
 	for _, m := range tier {
 		state := p.Modules[m]
 		if state == nil || state.SentAt != nil || !rollsOut(m) {
@@ -452,17 +605,64 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 		if err != nil {
 			return false, err
 		}
+		policy, err := inv.UpgradeOf(ctx, m)
+		if err != nil {
+			return false, err
+		}
+		var reports []inventory.Reported
+		if !policy.Together {
+			// Read for the choice of the first machine as well as for its report.
+			if reports, err = inv.LastReports(ctx); err != nil {
+				return false, err
+			}
+		}
 		now := time.Now().UTC()
-		state.SentAt = &now
+		step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
-		if len(running) == 0 {
+		switch {
+		case step.failed != "":
+			state.Why = step.failed
+			p.State = inventory.PlanFailed
+			p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was",
+				m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", ")))
+			fmt.Printf("%s: %s\n", p.ID, p.Note)
+			return true, nil
+		case step.waiting != "":
+			pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
+			continue
+		case len(step.send) == 0:
+			// No machine runs it: nothing to send, and nothing to wait for.
+			state.SentAt = &now
 			continue
 		}
-		if err := sendTo(ctx, open, running); err != nil {
+		// What sendToEach answers, not what was asked: the machine holding the bus is sent before
-			return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(running, ", "), p.Tier, err)
+		// the first when its user list must change (issue 249), and the plan waits for it too.
+		sent, err := sendToEach(ctx, open, step.send)
+		if err != nil {
+			// Not marked sent, so the next step tries again (issue 249): a grant that could not be
+			// issued is a send that did not happen.
+			return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
 		}
-		fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(running, ", "))
+		if step.first {
+			state.First = sent
+			state.FirstAt = &now
+			p.State = inventory.PlanRolling
+			p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
+			fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
+				p.ID, p.Tier, m, strings.Join(sent, ", "))
+			return true, nil
+		}
+		state.SentAt = &now
+		fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(sent, ", "))
 		return true, nil
 	}
+	if len(pending) > 0 {
+		note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") +
+			" to report it applied before the rest are sent"
+		changed := p.State != inventory.PlanRolling || p.Note != note
+		p.State = inventory.PlanRolling
+		p.Note = note
+		return changed, nil
+	}
 	// And wait for what the next tier needs running.
 	needed := gates(*p, edges, rollsOut)
 	if len(needed) > 0 {
@@ -489,10 +689,24 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			if state.BuiltAt != nil {
 				since = *state.BuiltAt
 			}
-			if state.SentAt != nil && state.SentAt.After(since) {
+			// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
-				since = *state.SentAt
+			// a module is sent twice — the first machine, then the rest — and SentAt is the second.
+			// Asked of every machine, the first machine's report, made between the two sends, read as
+			// older than the build, and the gate waited for a report it already had, for ever.
+			sinceFor := func(node string) time.Time {
+				at := since
+				sent := state.SentAt
+				for _, n := range state.First {
+					if n == node {
+						sent = state.FirstAt
+					}
+				}
+				if sent != nil && sent.After(at) {
+					at = *sent
+				}
+				return at
 			}
-			if ok, on := applied(m, since, running, reports); !ok {
+			if ok, on := appliedEach(m, sinceFor, running, reports); !ok {
 				waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
 			}
 		}
@@ -513,6 +727,110 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	return true, nil
 }
 
+// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249).
+type rolloutStep struct {
+	// send is the machines to send now; first, whether they are the first machine's send.
+	send  []string
+	first bool
+	// waiting names the first machines whose report the rest wait for.
+	waiting string
+	// failed says how a first machine did not take it; rest is what is then left alone.
+	failed string
+	rest   []string
+}
+
+// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218).
+//
+// Together, every machine running it at once, as the policy says. Otherwise one machine first — the
+// first by name among those that have reported within the bound, so a laptop that is away is not
+// the one the rest wait on; the first by name when none has; the same choice on every controller and
+// every resume — and the rest once each machine the first send reached reports, about the
+// declaration it was last sent, that it applied it. Compared by the store's own record of what was
+// sent (Reported.Current), never by this controller's clock against the machine's.
+//
+// **A first machine that fails, refuses, or does not report within the bound stops the rollout
+// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a
+// rollout: it held the plan open for ever, read as work in progress (issue 254).
+func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported,
+	now time.Time, bound time.Duration) rolloutStep {
+	if len(running) == 0 {
+		return rolloutStep{}
+	}
+	if together {
+		return rolloutStep{send: running}
+	}
+	byNode := map[string]inventory.Reported{}
+	for _, r := range reports {
+		byNode[r.Node] = r
+	}
+	if s.FirstAt == nil {
+		sorted := append([]string{}, running...)
+		sort.Strings(sorted)
+		for _, n := range sorted {
+			if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound {
+				return rolloutStep{send: []string{n}, first: true}
+			}
+		}
+		return rolloutStep{send: sorted[:1], first: true}
+	}
+	sentFirst := map[string]bool{}
+	for _, n := range s.First {
+		sentFirst[n] = true
+	}
+	var rest []string
+	for _, n := range running {
+		if !sentFirst[n] {
+			rest = append(rest, n)
+		}
+	}
+	var waiting, failed []string
+	for _, n := range s.First {
+		r, said := byNode[n]
+		// Only a report about what it was last sent says anything about this build.
+		if !said || r.At == nil || !r.Current {
+			waiting = append(waiting, n)
+			continue
+		}
+		switch r.Outcome {
+		case inventory.OutcomeApplied:
+		case inventory.OutcomeFailed, inventory.OutcomeRefused:
+			failed = append(failed, n+" "+r.Outcome+" what it was sent")
+		default:
+			waiting = append(waiting, n)
+		}
+	}
+	if len(failed) > 0 {
+		return rolloutStep{failed: strings.Join(failed, "; "), rest: rest}
+	}
+	if len(waiting) > 0 {
+		if now.Sub(*s.FirstAt) > bound {
+			return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s",
+				strings.Join(waiting, ", "), bound), rest: rest}
+		}
+		return rolloutStep{waiting: strings.Join(waiting, ", ")}
+	}
+	return rolloutStep{send: rest}
+}
+
+// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249).
+// An announced move of a module a plan held was left to that plan; a plan that ends without
+// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing
+// saying so. A module whose rollout stopped at its first machine is not among them: that stop was
+// the point. Said once.
+func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
+	var unsent []string
+	for name, s := range p.Modules {
+		if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) {
+			unsent = append(unsent, name)
+		}
+	}
+	if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") {
+		return
+	}
+	sort.Strings(unsent)
+	p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
+}
+
 // planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
 func planTicker(ctx context.Context, open *stores) {
 	advancePlans(ctx, open)
@@ -529,15 +847,24 @@ func planTicker(ctx context.Context, open *stores) {
 }
 
 // planLine is one plan as `status` says it.
-func planLine(p inventory.Plan, now time.Time) string {
+func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
+
+// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
+// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
+func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
 	where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
 	switch p.State {
 	case inventory.PlanDone:
 		return fmt.Sprintf("%s %s done, %d tier(s)", p.Repository, short(p.Commit), len(p.Tiers))
 	case inventory.PlanFailed:
 		return fmt.Sprintf("%s %s FAILED at %s: %s", p.Repository, short(p.Commit), where, p.Note)
+	case inventory.PlanSuperseded:
+		return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
+	}
+	since := now.Sub(p.Updated).Round(time.Second)
+	if waiting, paused := pausedWaiting(p, pause, now); paused {
+		return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
 	}
-	since := now.Sub(p.Updated).Round(time.Minute)
 	late := ""
 	if since > planWaitBound {
 		late = " — LATE"
@@ -551,20 +878,49 @@ func planLine(p inventory.Plan, now time.Time) string {
 
 // planFailedBuild marks the module a failed build was for when the result names no module: by the
 // repository and path the plan's modules were asked at.
+//
+// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an
+// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched
+// to the module it was asked for exactly. Repository and path remain for a plan from before ids
+// were kept.
 func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
+	asked, _ := link.BuildAskedAt(result.ID)
+	if plans, err := open.inventory.OpenPlans(ctx); err == nil {
+		if module := moduleAskedAs(plans, result.ID); module != "" {
+			planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID)
+			return
+		}
+	}
 	entries, err := open.inventory.Catalogued(ctx)
 	if err != nil {
 		return
 	}
 	for _, e := range entries {
 		if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
-			asked, _ := link.BuildAskedAt(result.ID)
+			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID)
-			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked)
 			return
 		}
 	}
 }
 
+// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing.
+func moduleAskedAs(plans []inventory.Plan, id string) string {
+	if id == "" {
+		return ""
+	}
+	for _, p := range plans {
+		if p.Tier >= len(p.Tiers) {
+			continue
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if s := p.Modules[m]; s != nil && s.Build == id {
+				return m
+			}
+		}
+	}
+	return ""
+}
+
 func repositoryMatches(a, b string) bool {
 	trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
 	return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
@@ -583,7 +939,7 @@ type planStatus struct {
 	Late       bool      `json:"late"`
 }
 
-func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
+func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
 	out := make([]planStatus, 0, len(plans))
 	for _, p := range plans {
 		ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
@@ -594,6 +950,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 				ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
 			}
 			ps.Late = now.Sub(p.Updated) > planWaitBound
+			// Paused is a person's decision, not lateness (novox/hq ADR 0219).
+			if waiting, paused := pausedWaiting(p, pause, now); paused {
+				ps.Waiting, ps.Late = waiting, false
+			}
 		}
 		out = append(out, ps)
 	}
@@ -601,12 +961,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 }
 
 // openPlans is the open plans among the recent ones, and how many have waited past the bound.
-func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
+func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
 	var open []inventory.Plan
 	late := 0
 	for _, p := range plans {
 		if p.Open() {
 			open = append(open, p)
+			if _, paused := pausedWaiting(p, pause, time.Now()); paused {
+				continue
+			}
 			if time.Since(p.Updated) > planWaitBound {
 				late++
 			}
@@ -639,7 +1002,7 @@ func plansCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
 		for i, tier := range p.Tiers {
 			marker := " "
 			if i == p.Tier && p.Open() {
@@ -654,6 +1017,9 @@ func plansCommand(ctx context.Context, args []string) error {
 					if s.Commit != "" {
 						state += " from " + short(s.Commit)
 					}
+					if s.Build != "" && s.State != "built" {
+						state += " (" + s.Build + ")"
+					}
 					if s.Why != "" {
 						state += ": " + s.Why
 					}
@@ -666,7 +1032,28 @@ func plansCommand(ctx context.Context, args []string) error {
 	if *whatIf != "" {
 		return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
 	}
-	if len(positionals) == 2 && positionals[0] == "stop" {
+	// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
+	if len(positionals) == 2 && positionals[0] == "retry" {
+		said, err := retryPlan(ctx, open, positionals[1])
+		if err != nil {
+			return err
+		}
+		fmt.Println(said)
+		return nil
+	}
+	// `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one
+	// waiting on a report that cannot come — so it stops reading as work in progress. Marked failed
+	// with who ended it; what it asked still builds and registers.
+	if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
+		how := "stopped"
+		if positionals[0] == "close" {
+			how = "closed"
+		}
+		release, err := inv.HoldPlans(ctx, true)
+		if err != nil {
+			return err
+		}
+		defer release()
 		p, err := inv.PlanByID(ctx, positionals[1])
 		if err != nil {
 			return err
@@ -675,12 +1062,16 @@ func plansCommand(ctx context.Context, args []string) error {
 			return fmt.Errorf("%s is already %s", p.ID, p.State)
 		}
 		p.State = inventory.PlanFailed
-		p.Note = "stopped by hand at tier " + fmt.Sprint(p.Tier)
+		p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
+		sayUnsent(&p, func(m string) bool {
+			u, err := inv.UpgradeOf(ctx, m)
+			return err == nil && u.RollOut
+		})
 		if err := inv.SavePlan(ctx, p); err != nil {
 			return err
 		}
-		fmt.Printf("%s stopped at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
+		fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
-			p.ID, p.Tier, len(p.Tiers))
+			p.ID, how, p.Tier, len(p.Tiers))
 		return nil
 	}
 	plans, err := inv.RecentPlans(ctx, *limit)
@@ -691,8 +1082,9 @@ func plansCommand(ctx context.Context, args []string) error {
 		fmt.Println("no merge has produced a plan yet")
 		return nil
 	}
+	pause := buildSeatPause(ctx, inv, plans)
 	for _, p := range plans {
-		fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
 	}
 	return nil
 }
@@ -755,7 +1147,13 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string
 			how := "built; its policy records, so nothing is sent"
 			if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut {
 				running, _ := inv.Running(ctx, name)
+				reports, _ := inv.LastReports(ctx)
 				how = "built, then sent to " + orNone(strings.Join(running, ", "))
+				// One machine first unless the policy says together (novox/hq issue 249).
+				if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 {
+					how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it",
+						first.send[0])
+				}
 				rolls[name] = how
 			}
 			fmt.Printf("    %-22s %s\n", name, how)
@@ -788,7 +1186,12 @@ func splitList(s string) []string {
 // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
 // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
 // Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
-func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build) bool {
+//
+// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR
+// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is
+// found by nothing else.
+func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build,
+	byID map[string]inventory.Build) bool {
 	changed := false
 	for _, m := range tier {
 		s := p.Modules[m]
@@ -797,6 +1200,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 		}
 		var outcome *inventory.Build
 		for i := range recorded[m] {
+			// Asked under an id, only that id's record answers (ADR 0219), and it is looked up below.
+			if s.Build != "" {
+				break
+			}
 			b := recorded[m][i]
 			if b.At.Before(*s.AskedAt) {
 				break
@@ -808,6 +1215,9 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 			}
 			outcome = &b
 		}
+		if own, found := byID[s.Build]; s.Build != "" && found {
+			outcome = &own
+		}
 		if outcome == nil {
 			continue
 		}

cmd/mesh-controller/release_plan_test.go

@@ -145,7 +145,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 		// Only a build from before the ask: not this ask's outcome.
 		"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
 	}
-	if !settleFromRecords(&p, p.Tiers[0], records) {
+	if !settleFromRecords(&p, p.Tiers[0], records, nil) {
 		t.Fatal("nothing settled, though the controller's build is recorded after the ask")
 	}
 	if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
@@ -162,13 +162,13 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 	late := map[string][]inventory.Build{"postgres": {
 		{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
 	}}
-	if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" {
+	if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
 		t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
 	}
 	// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
 	late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
 		Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
-	if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" ||
+	if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
 		r.Modules["postgres"].Commit != "4bcd5f73" {
 		t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
 	}
@@ -176,8 +176,39 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 	// A failure recorded after the ask fails the plan, as hearing it would have.
 	q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
 		Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
-	settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}})
+	settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
 	if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
 		t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
 	}
 }
+
+// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
+// each machine is judged from its own send, not from the last one (novox/hq issue 256).
+func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
+	built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
+	firstSent := built.Add(31 * time.Second)
+	firstReported := built.Add(43 * time.Second)
+	restSent := built.Add(58 * time.Second)
+	restReported := built.Add(74 * time.Second)
+	reports := []inventory.Reported{
+		{Node: "ace", At: &firstReported},
+		{Node: "g14", At: &restReported},
+	}
+	since := func(node string) time.Time {
+		if node == "ace" {
+			return firstSent
+		}
+		return restSent
+	}
+	if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
+		t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
+	}
+	// The old reading, every machine from the last send, is what held the plan.
+	if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
+		t.Fatal("the single-moment reading should hold the first machine back")
+	}
+	early := built.Add(10 * time.Second)
+	if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
+		t.Fatal("a report from before the machine was sent opened the gate")
+	}
+}

cmd/mesh-controller/rollout.go

@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
 // A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
 // would bury the ones that matter under a list nobody can act on.
 func speaksOnTheBus(m catalogue.Manifest) bool {
-	return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
+	return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
 		len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
 }
 

cmd/mesh-controller/rollout_first_test.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
+// once that machine has reported it applied; a module whose policy says together goes everywhere at
+// once, as before.
+func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
+	running := []string{"novox", "ace", "g14"}
+	sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	now := sentAt.Add(5 * time.Minute)
+	bound := 30 * time.Minute
+	after := sentAt.Add(time.Minute)
+	next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
+		return nextRollout(s, running, together, reports, now, bound)
+	}
+
+	// Together: every machine at once.
+	if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
+		t.Fatalf("a together policy did not send every machine at once: %+v", step)
+	}
+
+	// Otherwise the first by name, alone, when none has reported lately.
+	step := next(inventory.PlanModule{}, running, false, nil)
+	if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
+		t.Fatalf("the first send was %+v, wanted ace alone", step)
+	}
+
+	state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
+	report := func(outcome string, current bool) []inventory.Reported {
+		return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
+			{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
+	}
+
+	// No report yet, or one about an older declaration than it was last sent: wait.
+	for what, reports := range map[string][]inventory.Reported{
+		"no report":            nil,
+		"a report about older": report(inventory.OutcomeApplied, false),
+	} {
+		step := next(state, running, false, reports)
+		if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
+			t.Errorf("%s: %+v, wanted to wait for ace", what, step)
+		}
+	}
+
+	// Applied what it was last sent: the rest, and only the rest.
+	step = next(state, running, false, report(inventory.OutcomeApplied, true))
+	if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
+		t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
+	}
+
+	// Failed or refused: stop, the rest untouched.
+	for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
+		step := next(state, running, false, report(outcome, true))
+		if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
+			!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
+			t.Errorf("a first machine that %s it: %+v", outcome, step)
+		}
+	}
+
+	// The machine holding the bus went with the first send: the rest wait for it too, and it is
+	// not sent again.
+	both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
+	half := report(inventory.OutcomeApplied, true)
+	if step := next(both, running, false, half); step.waiting != "novox" {
+		t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
+	}
+	all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
+	if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
+		t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
+	}
+
+	// One machine, or none: nothing is waited for that cannot come.
+	if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
+		t.Fatalf("a module nothing runs was sent: %+v", step)
+	}
+	if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
+		t.Fatalf("a module on one machine waited for more: %+v", step)
+	}
+}
+
+// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
+// naming the machine and the bound; the rest are left alone.
+func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
+	sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
+	step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
+	if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
+		!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
+		t.Fatalf("a silent first machine: %+v", step)
+	}
+}
+
+// The first machine is the first by name among those heard from lately: a laptop that is away is
+// not the one the rest wait on. When none has been heard from, the first by name.
+func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
+	now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
+	reports := []inventory.Reported{
+		{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
+	}
+	step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
+	if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
+		t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
+	}
+}
+
+// An announced move of a module an open plan is still rolling out is left to the plan: sending it
+// here as well put the bundle on every machine at once (novox/hq issue 249).
+func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
+	sent := time.Now()
+	plans := []inventory.Plan{
+		{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
+		{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
+			"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
+	}
+	if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
+		t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
+	}
+	for _, m := range []string{"gitea", "keycloak"} {
+		if got := rolledOutByAPlan(plans, m); got != "" {
+			t.Errorf("%s, which no plan will send, was left to %s", m, got)
+		}
+	}
+}
+
+// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
+// stopped at its first machine is not among them.
+func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
+	at := time.Now()
+	p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
+		Modules: map[string]*inventory.PlanModule{
+			"agent":   {State: "built"},
+			"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
+			"sent":    {State: "built", SentAt: &at},
+			"notes":   {State: "built"},
+			"later":   {},
+		}}
+	rollsOut := func(m string) bool { return m != "notes" }
+	sayUnsent(&p, rollsOut)
+	sayUnsent(&p, rollsOut)
+	if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
+		t.Fatalf("the note reads %q", p.Note)
+	}
+}

cmd/mesh-controller/seat_dependencies_test.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
+// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
+// dependents, and `status` reports what composition does not yet refuse.
+
+func serviceManagerHolder() catalogue.Manifest {
+	return catalogue.Manifest{Module: "systemd", Version: "1",
+		Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
+			Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
+		Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
+}
+
+func packageManagerHolder() catalogue.Manifest {
+	return catalogue.Manifest{Module: "pacman", Version: "1",
+		Claims:    []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
+		Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
+}
+
+func aDaemon() catalogue.Manifest {
+	return catalogue.Manifest{Module: "sshd", Version: "1",
+		Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
+}
+
+func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, serviceManagerHolder())
+	register(t, open, packageManagerHolder())
+	register(t, open, aDaemon())
+
+	_, err := assign(ctx, open, "laptop", "sshd")
+	if err == nil {
+		t.Fatal("sshd went onto a machine nothing holds the service manager of")
+	}
+	for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not say %q:\n%v", want, err)
+		}
+	}
+	assigned, err := open.inventory.Assigned(ctx, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if contains(assigned, "sshd") {
+		t.Fatalf("a refused assignment was kept: %v", assigned)
+	}
+
+	// The holders depend on each other, so neither goes on alone — and both go on in one act.
+	if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
+		t.Fatal("systemd went on alone though its package needs a package manager")
+	}
+	if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
+		t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
+	}
+	if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
+		t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
+	}
+}
+
+func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
+	argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(argv, " ") != "assign laptop systemd pacman" {
+		t.Errorf("the seat's assign became %v", argv)
+	}
+}
+
+func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, serviceManagerHolder())
+	register(t, open, packageManagerHolder())
+	register(t, open, aDaemon())
+	if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := unassign(ctx, open, "laptop", "systemd")
+	if err == nil {
+		t.Fatal("the service manager came off a machine still running services")
+	}
+	for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not name %q:\n%v", want, err)
+		}
+	}
+	assigned, _ := open.inventory.Assigned(ctx, "laptop")
+	if !contains(assigned, "systemd") {
+		t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
+	}
+	if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
+		t.Fatalf("a dependent could not come off: %v", err)
+	}
+}
+
+func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
+	// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
+	defer catalogue.EnforcingSeatDependencies(false)()
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, serviceManagerHolder())
+	register(t, open, aDaemon())
+	// Assigned straight into the store: a machine whose modules predate the rule, which is every
+	// machine on the day it ships.
+	if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
+		t.Fatal(err)
+	}
+
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, refused := asked.refused["laptop"]; refused {
+		t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
+	}
+	if asked.well() {
+		t.Error("a mesh with an unheld dependency reads as all well")
+	}
+	got := printed(t, func() error { return printStatus(asked) })
+	for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("status does not say %q:\n%s", want, got)
+		}
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Unheld []catalogue.Unheld `json:"unheld"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil {
+		t.Fatal(err)
+	}
+	if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
+		t.Errorf("the document's unheld is %+v", doc.Unheld)
+	}
+}

cmd/mesh-controller/seats.go

@@ -29,11 +29,13 @@ type seatHolder struct {
 
 // seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
 type seatRow struct {
-	Seat     string       `json:"seat"`
+	Seat     string `json:"seat"`
-	Scope    string       `json:"scope"`
+	Scope    string `json:"scope"`
-	Delivers string       `json:"delivers,omitempty"`
+	Delivers string `json:"delivers,omitempty"`
-	Decision string       `json:"decision"`
+	Decision string `json:"decision"`
-	Holders  []seatHolder `json:"holders"`
+	// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
+	Replicated bool         `json:"replicated,omitempty"`
+	Holders    []seatHolder `json:"holders"`
 }
 
 // seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
 	rows := make([]seatRow, 0, len(seats))
 	for _, s := range seats {
 		row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
-			Holders: []seatHolder{}}
+			Replicated: s.Replicated, Holders: []seatHolder{}}
 		seen := map[seatHolder]bool{}
 		for _, h := range held {
 			// Resolve the held claim to a seat rather than comparing names, so a record naming a
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 	if len(args) == 3 && args[1] == "--to" {
-		return handOver(ctx, args[0], args[2])
+		return handOver(ctx, args[0], args[2], false)
 	}
-	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>")
+	if len(args) == 3 && args[1] == "--add" {
+		return handOver(ctx, args[0], args[2], true)
+	}
+	return fmt.Errorf("seat rename <from> <to>  |  seat <name> --to <node>/<module>  |  " +
+		"seat <name> --add <node>/<module>")
 }
 
 // handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
 // **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
 // and refusing to record a handover to a module the node has not started would make the handover
 // impossible to do before the switch instead of as the switch.
-func handOver(ctx context.Context, seatName, to string) error {
+//
+// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
+// records the named assignment as a further holder and leaves every holder on record as it is. A
+// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
+// the one named, as it always did.
+func handOver(ctx context.Context, seatName, to string, adding bool) error {
 	nodeName, module, ok := strings.Cut(to, "/")
 	if !ok || nodeName == "" || module == "" {
 		return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
 	if !known {
 		return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
 	}
+	if adding && !seat.Replicated {
+		return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
+			"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
+	}
 	assigned, err := inv.Assigned(ctx, nodeName)
 	if err != nil {
 		return err
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 		return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
 	}
 	var was string
+	var held []string
 	holdings, err := inv.Holdings(ctx)
 	if err != nil {
 		return err
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
 	for _, h := range holdings {
 		if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
 			was = h.Node
+			held = append(held, h.Node)
 		}
 	}
 
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
 	} else if err := catalogue.CanHold(*m, seat); err != nil {
 		return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
 	}
+	if adding {
+		if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
+			return err
+		}
+		fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
+			strings.Join(held, ", "))
+		fmt.Printf("  `push --behind` re-declares every machine that reads the seat's holders\n")
+		return nil
+	}
 	if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
 		return err
 	}

cmd/mesh-controller/seatverbs.go

@@ -36,19 +36,194 @@ type verbAnswer struct {
 
 // argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
 // only the arguments each declares: a caller cannot reach a flag the schema did not name.
+//
+// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
+// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
+// the verb declares but did not use for the command line it composed — given beside another that
+// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
+// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
+// not take that" would have stopped it before anything was sent.
 func argvFor(verb string, args map[string]any) ([]string, error) {
-	str := func(key string) string {
+	a, err := readArguments(verb, args)
-		v, _ := args[key].(string)
+	if err != nil {
-		return strings.TrimSpace(v)
+		return nil, err
 	}
-	need := func(keys ...string) error {
+	argv, err := a.commandLine()
-		for _, k := range keys {
+	if len(a.misread) > 0 {
-			if str(k) == "" {
+		// The table and the command line disagree: the verb reads an argument no caller can see
-				return fmt.Errorf("%s needs %q", verb, k)
+		// in its schema, so no caller could ever pass it.
+		return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
+			"table and its command lines disagree", verb, quoteAll(a.misread))
+	}
+	if err != nil {
+		return nil, err
+	}
+	if unused := a.unused(); len(unused) > 0 {
+		return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
+			"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
+	}
+	return argv, nil
+}
+
+// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
+// command line was composed from.
+type verbArguments struct {
+	verb     string
+	given    map[string]string
+	used     map[string]bool
+	declared map[string]bool
+	misread  []string // arguments the command line read that the schema does not declare: a bug here
+}
+
+// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
+// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
+// wrote.
+func controllerVerb(name string) (catalogue.Verb, bool) {
+	for _, v := range catalogue.ControllerVerbs {
+		if v.Name == name {
+			return v, true
+		}
+	}
+	return catalogue.Verb{}, false
+}
+
+// declaredArguments are a schema's properties, and which of them are switches.
+func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
+	switches = map[string]bool{}
+	props, _ := v.Input["properties"].(map[string]any)
+	for name, p := range props {
+		names = append(names, name)
+		desc, _ := p.(map[string]any)
+		switch enum := desc["enum"].(type) {
+		case []string:
+			switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
+		case []any:
+			switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
+		}
+	}
+	sort.Strings(names)
+	return names, switches
+}
+
+// readArguments refuses what the verb does not take, before anything is composed.
+func readArguments(verb string, args map[string]any) (*verbArguments, error) {
+	v, known := controllerVerb(verb)
+	if !known {
+		return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
+	}
+	names, switches := declaredArguments(v)
+	declared := map[string]bool{}
+	for _, n := range names {
+		declared[n] = true
+	}
+	takes := "none"
+	if len(names) > 0 {
+		takes = quoteAll(names)
+	}
+	a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
+	keys := make([]string, 0, len(args))
+	for k := range args {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if !declared[k] {
+			return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
+		}
+		var value string
+		switch x := args[k].(type) {
+		case nil:
+			continue
+		case string:
+			value = strings.TrimSpace(x)
+		case bool:
+			if !switches[k] {
+				return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
+			}
+			value = fmt.Sprint(x)
+		default:
+			return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
+		}
+		if switches[k] {
+			switch value {
+			case "true":
+			case "false", "":
+				continue // said and off: the same as not given, and nothing passed over
+			default:
+				return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
 			}
 		}
-		return nil
+		if value != "" {
+			a.given[k] = value
+		}
 	}
+	return a, nil
+}
+
+// str is one argument's value, marked as used.
+func (a *verbArguments) str(key string) string {
+	if !a.declared[key] {
+		a.misread = append(a.misread, key)
+	}
+	a.used[key] = true
+	return a.given[key]
+}
+
+// on is a switch, marked as used.
+func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
+
+// need refuses a call missing a required argument, in the verb's own words.
+func (a *verbArguments) need(keys ...string) error {
+	for _, k := range keys {
+		if a.str(k) == "" {
+			return fmt.Errorf("%s needs %q", a.verb, k)
+		}
+	}
+	return nil
+}
+
+// unused are the arguments given that the command line was not composed from.
+func (a *verbArguments) unused() []string {
+	var out []string
+	for k := range a.given {
+		if !a.used[k] {
+			out = append(out, k)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (a *verbArguments) usedGiven() []string {
+	var out []string
+	for k := range a.given {
+		if a.used[k] {
+			out = append(out, k)
+		}
+	}
+	sort.Strings(out)
+	if len(out) == 0 {
+		return []string{"nothing"}
+	}
+	return out
+}
+
+func quoteAll(xs []string) string {
+	q := make([]string, len(xs))
+	for i, x := range xs {
+		if x == "nothing" {
+			q[i] = x
+			continue
+		}
+		q[i] = fmt.Sprintf("%q", x)
+	}
+	return strings.Join(q, ", ")
+}
+
+// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
+// argument and then drops it would pass it over, which is what the check after it exists to refuse.
+func (a *verbArguments) commandLine() ([]string, error) {
+	verb, str, on, need := a.verb, a.str, a.on, a.need
 	switch verb {
 	case "command":
 		// The generic verb: the command line as given, split as a shell would split it, with
@@ -65,6 +240,8 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 			return nil, errors.New("command names no command")
 		}
 		return argv, nil
+	case "tools":
+		return nil, errors.New("tools is answered from the records, not by a command")
 	case "status":
 		return []string{"status", "--json"}, nil
 	case "nodes":
@@ -82,10 +259,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		if id := str("log"); id != "" {
 			return []string{"builds", "--log", id}, nil
 		}
-		if m := str("module"); m != "" {
+		argv := []string{"builds"}
-			return []string{"builds", m}, nil
+		if n := str("limit"); n != "" {
+			argv = append(argv, "-n", n)
 		}
-		return []string{"builds"}, nil
+		if m := str("module"); m != "" {
+			argv = append(argv, m)
+		}
+		return argv, nil
 	case "plans":
 		if r := str("repository"); r != "" {
 			argv := []string{"plans", "--what-if", r}
@@ -97,23 +278,69 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 			}
 			return argv, nil
 		}
-		if id := str("stop"); id != "" {
+		for _, act := range []string{"stop", "close", "retry"} {
-			return []string{"plans", "stop", id}, nil
+			if id := str(act); id != "" {
+				return []string{"plans", act, id}, nil
+			}
 		}
 		if id := str("id"); id != "" {
 			return []string{"plans", id}, nil
 		}
-		return []string{"plans"}, nil
+		argv := []string{"plans"}
+		if n := str("limit"); n != "" {
+			argv = append(argv, "-n", n)
+		}
+		return argv, nil
+	// The build queue (novox/hq ADR 0219).
+	case "queue":
+		return []string{"queue"}, nil
+	case "cancel", "kill":
+		if err := need("id"); err != nil {
+			return nil, err
+		}
+		return []string{verb, str("id")}, nil
+	case "clear":
+		if on("dead") {
+			return []string{"clear", "--dead"}, nil
+		}
+		return []string{"clear"}, nil
+	case "rebuild":
+		if err := need("what"); err != nil {
+			return nil, err
+		}
+		return []string{"rebuild", str("what")}, nil
+	case "replay":
+		if err := need("id"); err != nil {
+			return nil, err
+		}
+		argv := []string{"replay", str("id")}
+		if on("register") {
+			argv = append(argv, "--register")
+		}
+		if on("older") {
+			argv = append(argv, "--older")
+		}
+		return argv, nil
+	case "pause", "resume":
+		if n := str("node"); n != "" {
+			return []string{verb, n}, nil
+		}
+		return []string{verb}, nil
 	case "plan":
 		if err := need("node"); err != nil {
 			return nil, err
 		}
+		if on("files") {
+			return []string{"plan", str("node"), "--files"}, nil
+		}
 		return []string{"plan", str("node"), "--json"}, nil
 	case "assign", "unassign":
 		if err := need("node", "module"); err != nil {
 			return nil, err
 		}
-		return []string{verb, str("node"), str("module")}, nil
+		// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
+		// the seats that apply resources depend on each other and go on together.
+		return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
 	case "pin":
 		if err := need("node", "provision", "from", "module"); err != nil {
 			return nil, err
@@ -129,8 +356,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		// what a person at a shell does too. A tool call that blocked for a push's whole apply would
 		// time out on every machine that takes a minute, and say nothing about the ones that did not.
 		if n := str("node"); n != "" {
+			// behind is not read here: given with a machine, it is refused as passed over — naming
+			// a machine and asking for every machine behind are two requests, and guessing one
+			// would push a machine nobody named, or not push one somebody did.
 			return []string{"push", n, "--wait", "0"}, nil
 		}
+		// No machine: the whole mesh, whether or not behind said so. The command's answer says it
+		// first, so a caller who meant one machine reads that it was not one.
+		on("behind")
 		return []string{"push", "--behind", "--wait", "0"}, nil
 	case "rotate":
 		if p := str("provision"); p != "" {
@@ -140,11 +373,17 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 			}
 			return argv, nil
 		}
-		if str("node") != "" && str("module") != "" && str("secret") != "" {
+		_, node := a.given["node"]
+		_, module := a.given["module"]
+		_, secret := a.given["secret"]
+		if node || module || secret {
+			if err := need("node", "module", "secret"); err != nil {
+				return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
+			}
 			return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
 		}
-		// Half of either shape: the command says its usage, which names both shapes, and that is
+		// Neither shape: the command says its usage, which names both, and that is the answer the
-		// the answer the caller needs.
+		// caller needs.
 		return []string{"rotate"}, nil
 	case "settings":
 		// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
@@ -152,15 +391,16 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		if err := need("module"); err != nil {
 			return nil, err
 		}
-		argv := []string{"settings", "set", str("module")}
+		var argv []string
-		switch {
+		if on("clear") {
-		case str("clear") == "true":
 			argv = []string{"settings", "clear", str("module")}
-		case str("values") != "":
+		} else {
-			argv = append(argv, str("values"))
+			argv = []string{"settings", "set", str("module")}
+			// Neither values nor clear: the command says its usage, which names both.
+			if v := str("values"); v != "" {
+				argv = append(argv, v)
+			}
 		}
-		// Neither values nor clear: the command says its usage, which names both, and that is the
-		// answer the caller needs — the same as `rotate` given half of either shape.
 		if n := str("node"); n != "" {
 			argv = append(argv, "--node", n)
 		}
@@ -192,11 +432,12 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		}
 		return argv, nil
 	}
-	return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
+	return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
+		verb, catalogue.ControllerSeatName)
 }
 
 // jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
-var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
+var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
 
 // runVerb runs this binary with the given command line and gathers what it said.
 func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
@@ -245,7 +486,17 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 	for _, v := range seat.Serves {
 		verb := v.Name
 		if verb == "tools" {
-			handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
+			handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
+				args := map[string]any{}
+				if len(bytes.TrimSpace(raw)) > 0 {
+					if err := json.Unmarshal(raw, &args); err != nil {
+						return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
+					}
+				}
+				// Refused like any verb's: tools takes nothing, and something given is not ignored.
+				if _, err := readArguments(verb, args); err != nil {
+					return nil, err
+				}
 				return seatTools(), nil
 			}
 			continue
@@ -310,9 +561,10 @@ func seatTools() map[string]any {
 }
 
 // sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
-// verb runnable rather than that it happens to want the arguments the check guessed.
+// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
+// more, since an argument a verb does not declare is refused.
 func sampleArguments(v catalogue.Verb) map[string]any {
-	sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
+	sample := map[string]any{}
 	switch required := v.Input["required"].(type) {
 	case []string:
 		for _, k := range required {

cmd/mesh-controller/seatverbs_schema_test.go

@@ -0,0 +1,365 @@
+package main
+
+import (
+	"encoding/json"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The schemas the controller serves, verb by verb, as the console receives them: from the
+// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
+func servedSchemas(t *testing.T) map[string]catalogue.Verb {
+	t.Helper()
+	handlers, behind, err := seatToolHandlers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(behind) != 0 {
+		t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
+	}
+	served := map[string]catalogue.Verb{}
+	for _, e := range seatAnnouncement(handlers).Endpoints {
+		var input map[string]any
+		if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
+			t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
+		}
+		served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
+	}
+	if len(served) != len(catalogue.ControllerVerbs) {
+		t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
+	}
+	return served
+}
+
+// subsetsOf is every subset of the names, the empty one included.
+func subsetsOf(names []string) [][]string {
+	var out [][]string
+	for mask := 0; mask < 1<<len(names); mask++ {
+		var s []string
+		for i, n := range names {
+			if mask&(1<<i) != 0 {
+				s = append(s, n)
+			}
+		}
+		out = append(out, s)
+	}
+	return out
+}
+
+func argumentsFor(subset []string, switches map[string]bool) map[string]any {
+	args := map[string]any{}
+	for _, n := range subset {
+		if switches[n] {
+			args[n] = "true"
+		} else {
+			args[n] = "x-" + n
+		}
+	}
+	return args
+}
+
+// saidOutright are the switches that say the default aloud, so the line is the same without them —
+// on purpose, and only these.
+var saidOutright = map[string]string{
+	"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
+}
+
+// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
+// every combination of the arguments its schema declares, the verb either refuses the call, or
+// composes a command line that each given argument changed: taking any one away changes the line
+// or makes it refused. An argument the line is the same without is one the verb ignored — the
+// shape of the push that named a machine and pushed every machine behind.
+func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
+	for name, v := range servedSchemas(t) {
+		if name == "tools" {
+			continue
+		}
+		names, switches := declaredArguments(v)
+		for _, subset := range subsetsOf(names) {
+			args := argumentsFor(subset, switches)
+			argv, err := argvFor(name, args)
+			if err != nil {
+				if strings.Contains(err.Error(), "does not declare") {
+					t.Errorf("%s %v: %v", name, args, err)
+				}
+				continue
+			}
+			for _, dropped := range subset {
+				fewer := map[string]any{}
+				for k, val := range args {
+					if k != dropped {
+						fewer[k] = val
+					}
+				}
+				without, err := argvFor(name, fewer)
+				if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
+					t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
+				}
+			}
+		}
+	}
+}
+
+// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
+// what it requires and one argument more; and `node` in particular, for every verb whose schema
+// does not take a machine.
+func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
+	for name, v := range servedSchemas(t) {
+		names, _ := declaredArguments(v)
+		strangers := []string{"no-such-argument"}
+		if !contains(names, "node") {
+			strangers = append(strangers, "node")
+		}
+		for _, stranger := range strangers {
+			args := sampleArguments(v)
+			args[stranger] = "x"
+			_, err := argvFor(name, args)
+			if name == "tools" {
+				_, err = readArguments(name, args)
+			}
+			if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
+				t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
+			}
+		}
+	}
+	if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
+		t.Error("a switch took a word that is neither true nor false, and would have read it as false")
+	}
+	if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
+		t.Error("a number was taken as a machine's name, or as no machine")
+	}
+	if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
+		t.Errorf("a switch given as JSON true: %v %v", argv, err)
+	}
+}
+
+// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
+// naming one beside behind is refused rather than one of the two guessed.
+func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
+	argv, err := argvFor("push", map[string]any{"node": "g1"})
+	if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
+		t.Fatalf("a named push: %v %v", argv, err)
+	}
+	for _, args := range []map[string]any{{}, {"behind": "true"}} {
+		argv, err := argvFor("push", args)
+		if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
+			t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
+		}
+	}
+	if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
+		!strings.Contains(err.Error(), `"behind"`) {
+		t.Fatalf("a named push with behind was taken: %v", err)
+	}
+	if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
+		t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
+	}
+}
+
+// commandFlags is every flag set this package's commands parse, by the name the set is made with,
+// and the flags defined on it — read from the source, so a flag added to a command is seen here
+// without anyone remembering to.
+func commandFlags(t *testing.T) map[string][]string {
+	t.Helper()
+	files, err := filepath.Glob("*.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fset := token.NewFileSet()
+	out := map[string][]string{}
+	for _, f := range files {
+		if strings.HasSuffix(f, "_test.go") {
+			continue
+		}
+		file, err := parser.ParseFile(fset, f, nil, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, decl := range file.Decls {
+			fn, ok := decl.(*ast.FuncDecl)
+			if !ok || fn.Body == nil {
+				continue
+			}
+			sets := map[string]string{} // variable → the set's name
+			ast.Inspect(fn.Body, func(n ast.Node) bool {
+				if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
+					if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
+						if id, ok := assign.Lhs[0].(*ast.Ident); ok {
+							if name, ok := stringLit(call.Args[0]); ok {
+								sets[id.Name] = name
+								out[name] = append(out[name], []string{}...)
+							}
+						}
+					}
+				}
+				call, ok := n.(*ast.CallExpr)
+				if !ok {
+					return true
+				}
+				sel, ok := call.Fun.(*ast.SelectorExpr)
+				if !ok {
+					return true
+				}
+				recv, ok := sel.X.(*ast.Ident)
+				if !ok || sets[recv.Name] == "" {
+					return true
+				}
+				arg := 0
+				switch sel.Sel.Name {
+				case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
+				case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
+					arg = 1
+				default:
+					return true
+				}
+				if arg < len(call.Args) {
+					if flagName, ok := stringLit(call.Args[arg]); ok {
+						out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
+					}
+				}
+				return true
+			})
+		}
+	}
+	return out
+}
+
+func isSelector(e ast.Expr, pkg, name string) bool {
+	sel, ok := e.(*ast.SelectorExpr)
+	if !ok {
+		return false
+	}
+	id, ok := sel.X.(*ast.Ident)
+	return ok && id.Name == pkg && sel.Sel.Name == name
+}
+
+func stringLit(e ast.Expr) (string, bool) {
+	lit, ok := e.(*ast.BasicLit)
+	if !ok || lit.Kind != token.STRING {
+		return "", false
+	}
+	s, err := strconv.Unquote(lit.Value)
+	return s, err == nil
+}
+
+// accountedFlags are the flags of a verb's command that are not an argument of the same name:
+// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
+// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
+var accountedFlags = map[string]map[string]string{
+	"status": {"json": "set by the verb: the answer is data"},
+	"seats":  {"json": "set by the verb: the answer is data"},
+	"queue":  {"json": "not set: the verb answers the table a person reads"},
+	"plan":   {"json": "set by the verb unless files is asked"},
+	"push":   {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
+	"build": {
+		"wait":    "set by the verb to 0: the id follows the build (issue 176)",
+		"self":    "set by the verb from the repository's form: a path on the forge, or a URL",
+		"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
+		"behind":  "withheld: the named verb builds one named repository; `command` reaches the rest",
+		"on":      "withheld: the named verb builds one named repository; `command` reaches the rest",
+	},
+	"builds": {"n": "=limit"},
+	"plans":  {"n": "=limit", "what-if": "=repository"},
+}
+
+// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
+// from the source, so a flag added to a command — or a verb added whose command takes flags —
+// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
+// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
+// reads.
+func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
+	flags := commandFlags(t)
+	if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
+		t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
+	}
+	reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
+	served := servedSchemas(t)
+	for name, v := range served {
+		if name == "tools" || name == "command" {
+			continue
+		}
+		names, switches := declaredArguments(v)
+		for _, subset := range subsetsOf(names) {
+			argv, err := argvFor(name, argumentsFor(subset, switches))
+			if err != nil {
+				continue
+			}
+			// The flag set is named by the longest run of leading words that names one.
+			var words []string
+			for _, w := range argv {
+				if strings.HasPrefix(w, "-") {
+					break
+				}
+				words = append(words, w)
+			}
+			for n := len(words); n > 0; n-- {
+				if _, has := flags[strings.Join(words[:n], " ")]; has {
+					if reached[name] == nil {
+						reached[name] = map[string]bool{}
+					}
+					reached[name][strings.Join(words[:n], " ")] = true
+					break
+				}
+			}
+		}
+	}
+	used := map[string]map[string]bool{}
+	verbs := make([]string, 0, len(reached))
+	for verb := range reached {
+		verbs = append(verbs, verb)
+	}
+	sort.Strings(verbs)
+	for _, verb := range verbs {
+		declared, _ := declaredArguments(served[verb])
+		for set := range reached[verb] {
+			if used[set] == nil {
+				used[set] = map[string]bool{}
+			}
+			for _, flagName := range flags[set] {
+				why, accounted := accountedFlags[set][flagName]
+				switch {
+				case accounted && strings.HasPrefix(why, "="):
+					used[set][flagName] = true
+					if !contains(declared, strings.TrimPrefix(why, "=")) {
+						t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
+							verb, flagName, set, strings.TrimPrefix(why, "="))
+					}
+				case accounted:
+					used[set][flagName] = true
+				case contains(declared, flagName):
+				default:
+					t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
+						"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
+				}
+			}
+		}
+	}
+	for set, fs := range accountedFlags {
+		for flagName := range fs {
+			if !used[set][flagName] {
+				t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
+			}
+		}
+	}
+}
+
+// Every verb's required arguments are properties of its schema: a schema that requires what it
+// does not describe is the uncallable verb of issue 244 from the other side.
+func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
+	for name, v := range servedSchemas(t) {
+		names, _ := declaredArguments(v)
+		for k := range sampleArguments(v) {
+			if !contains(names, k) {
+				t.Errorf("%s requires %q and does not describe it", name, k)
+			}
+		}
+	}
+}

cmd/mesh-controller/seatverbs_test.go

@@ -10,29 +10,6 @@ import (
 	"github.com/novox/mesh-controller/internal/catalogue"
 )
 
-// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
-// schema names and no other (novox/hq ADR 0154, ADR 0035).
-func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
-	for _, v := range catalogue.ControllerVerbs {
-		if v.Name == "tools" {
-			continue
-		}
-		args := map[string]any{}
-		props, _ := v.Input["properties"].(map[string]any)
-		for name := range props {
-			args[name] = "x"
-		}
-		argv, err := argvFor(v.Name, args)
-		if err != nil {
-			t.Errorf("%s: %v", v.Name, err)
-			continue
-		}
-		if argv[0] == "" {
-			t.Errorf("%s: empty command", v.Name)
-		}
-	}
-}
-
 // `builds` given a build's id reads that build's log from the bus rather than listing builds
 // (novox/hq ADR 0157).
 func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
@@ -70,9 +47,14 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
 	if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
 		t.Fatalf("an own secret: %v", argv)
 	}
-	argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
+	if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
-	if strings.Join(argv, " ") != "rotate" {
+		t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
-		t.Fatalf("half an own secret falls to the command's usage: %v", argv)
+	}
+	if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
+		t.Fatalf("neither shape falls to the command's usage: %v", argv)
+	}
+	if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
+		t.Fatal("both shapes at once were taken, and one of them passed over")
 	}
 }
 

cmd/mesh-controller/sendable.go

@@ -31,6 +31,10 @@ type sendable struct {
 	// the same composition as its received files, and every machine's private-network address.
 	Received map[string]map[string][]catalogue.Contribution
 	Mesh     []string
+	// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
+	// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
+	// whether that machine must go first is read from the list alone (novox/hq issue 249).
+	BusUsers string
 	// LeftOut is every module of the machine's set left out of this declaration because a stored
 	// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
 	// keeps that module's held things and touches none of its containers; a machine is told
@@ -39,6 +43,10 @@ type sendable struct {
 	LeftOut []string
 	// leftOutWhy is why each was, for push and plan to say; never on the wire.
 	leftOutWhy map[string]string
+	// Builds is the build of each module this declaration carries — module to the commit its build
+	// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
+	// Composed only on the send path; nil records that it is not known.
+	Builds map[string]string
 }
 
 // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on

cmd/mesh-controller/standing.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
+//
+// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
+// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
+// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
+// provider now announces a consumer it has failed for minutes; the controller keeps it until the
+// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
+
+// standings keeps what providers say, in the inventory.
+type standings struct{ inv *inventory.Inventory }
+
+func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
+	return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
+		Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
+		Consumer: st.Consumer, ConsumerNode: st.Node,
+		Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
+	})
+}
+
+// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
+//
+// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
+// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
+// consumer clears it.
+func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
+	all, err := inv.FailingProviders(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
+	}
+	assigned := map[string]map[string]bool{}
+	var out []inventory.ProviderStanding
+	for _, s := range all {
+		on, asked := assigned[s.ProviderNode]
+		if !asked {
+			modules, err := inv.Assigned(ctx, s.ProviderNode)
+			if err != nil {
+				// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
+				modules = nil
+			}
+			on = map[string]bool{}
+			for _, m := range modules {
+				on[m] = true
+			}
+			assigned[s.ProviderNode] = on
+		}
+		if on[s.Module] {
+			out = append(out, s)
+		}
+	}
+	return out, nil
+}
+
+// failingLines is how status says them: one consumer per entry, the error under it, and a provider
+// that stopped repeating itself said so.
+func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
+	var out []string
+	for _, s := range list {
+		where := s.Module
+		if s.ProviderNode != "" {
+			where += " on " + s.ProviderNode
+		}
+		whom := s.Consumer
+		if s.ConsumerNode != "" {
+			whom += " (" + s.ConsumerNode + ")"
+		}
+		out = append(out, fmt.Sprintf("  %-24s fails %s: %s, for %s (%d attempts since %s)",
+			where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
+			s.Since.Local().Format("2006-01-02 15:04")))
+		if e := strings.TrimSpace(s.Error); e != "" {
+			out = append(out, fmt.Sprintf("  %-24s %s", "", firstLine(e)))
+		}
+		if s.Quiet(now) {
+			out = append(out, fmt.Sprintf("  %-24s not said again for %s — the provider has stopped "+
+				"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
+		}
+	}
+	return out
+}
+
+func orUnclassed(class string) string {
+	if class == "" {
+		return "failing"
+	}
+	return class
+}
+
+// printFailing is the status section, said when there is anything to say.
+func printFailing(list []inventory.ProviderStanding, now time.Time) {
+	if len(list) == 0 {
+		return
+	}
+	fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
+	for _, line := range failingLines(list, now) {
+		fmt.Println(line)
+	}
+	fmt.Printf("\n  the provider's journal has every attempt; it says recovered on its next success\n\n")
+}
+
+// failingOn is the standings that concern one machine: a provider running there, or a consumer.
+func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
+	var out []inventory.ProviderStanding
+	for _, s := range list {
+		if s.ProviderNode == node || s.ConsumerNode == node {
+			out = append(out, s)
+		}
+	}
+	return out
+}

cmd/mesh-controller/standing_test.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
+// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
+// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
+func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
+		Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
+	if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
+		t.Fatal(err)
+	}
+	kept := standings{open.inventory}
+	since := time.Now().Add(-23 * time.Hour)
+	failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
+		Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
+		Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
+	if _, err := kept.Stood(ctx, failing); err != nil {
+		t.Fatal(err)
+	}
+
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if asked.well() {
+		t.Fatal("a mesh whose identity provider fails a consumer reads as well")
+	}
+	said := printed(t, func() error { return printStatus(asked) })
+	for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
+		"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
+		if !strings.Contains(said, want) {
+			t.Fatalf("status does not say %q:\n%s", want, said)
+		}
+	}
+	if strings.Contains(said, "all doing what they were told") {
+		t.Fatalf("status said all well beside a failing provider:\n%s", said)
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Failing []inventory.ProviderStanding `json:"failing"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
+		t.Fatalf("the document does not carry it: %v\n%s", err, body)
+	}
+	// Both machines' `node show` name it: where the provider runs, and where the consumer is.
+	for _, node := range []string{"anchor", "laptop"} {
+		shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
+		if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
+			t.Fatalf("node show %s does not name it:\n%s", node, shown)
+		}
+	}
+
+	// Recovered: gone, and the mesh may be well again as far as this is concerned.
+	failing.Failing = false
+	if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
+		t.Fatalf("%v %v", cleared, err)
+	}
+	asked, err = theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
+	}
+}
+
+// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
+// not a problem.
+func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
+		ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("%+v", asked.failing)
+	}
+}
+
+func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
+	now := time.Now()
+	lines := strings.Join(failingLines([]inventory.ProviderStanding{{
+		Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
+		Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
+	}}, now), "\n")
+	for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
+		if !strings.Contains(lines, want) {
+			t.Fatalf("%q not in:\n%s", want, lines)
+		}
+	}
+	if strings.Contains(lines, "more") {
+		t.Fatalf("more than the first line of an error:\n%s", lines)
+	}
+}

cmd/mesh-controller/status.go

@@ -129,6 +129,10 @@ func printStatus(asked answers) error {
 		fmt.Println()
 	}
 
+	// A provider failing a consumer, beside machines failing what they were told: both are something
+	// not working now (novox/hq ADR 0224).
+	printFailing(asked.failing, time.Now())
+
 	if len(quiet) > 0 {
 		var said []string
 		for _, n := range quiet {
@@ -138,14 +142,14 @@ func printStatus(asked answers) error {
 			len(quiet), strings.Join(said, "\n  "))
 	}
 
-	if open, late := openPlans(asked.plans); len(open) > 0 {
+	if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
 		fmt.Printf("%d plan(s) open", len(open))
 		if late > 0 {
 			fmt.Printf(", %d waiting past %s", late, planWaitBound)
 		}
 		fmt.Println(":")
 		for _, p := range open {
-			fmt.Printf("  %s\n", planLine(p, time.Now()))
+			fmt.Printf("  %s\n", planLineWith(p, time.Now(), asked.paused))
 		}
 		fmt.Println()
 	}
@@ -282,6 +286,22 @@ func printStatus(asked answers) error {
 		fmt.Printf("\n  `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
 	}
 
+	if len(asked.unheld) > 0 {
+		// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
+		// is sent what it would be; this says which of its modules depend on a seat nothing there
+		// holds, until every machine has its holders and the switch makes it a refusal.
+		fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
+			len(asked.unheld))
+		for _, u := range asked.unheld {
+			holders := "no module in the catalogue claims it yet"
+			if len(u.Holders) > 0 {
+				holders = "could be held by " + strings.Join(u.Holders, ", ")
+			}
+			fmt.Printf("  %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
+		}
+		fmt.Printf("\n  `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
+	}
+
 	if adopted := adoptedNodes(nodes); len(adopted) > 0 {
 		// Said, because nothing forces the flip: a node left adopted is visible here rather than
 		// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -386,10 +406,32 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	if err != nil {
 		return answers{}, err
 	}
+	// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
+	// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
+	// the private network is built from and should say nothing else; a machine that does not
+	// resolve is already in refused, and is passed over here.
+	for _, n := range out.nodes {
+		plan, _, err := planFor(ctx, open, n.Name)
+		if err != nil {
+			if unresolvable(err) {
+				continue
+			}
+			return answers{}, err
+		}
+		out.unheld = append(out.unheld, plan.Unheld...)
+	}
+	// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
+	// providers announced: nothing else in the mesh knows whether a provision is being made.
+	out.failing, err = failingProviders(ctx, inv)
+	if err != nil {
+		return answers{}, err
+	}
 	out.plans, err = inv.RecentPlans(ctx, 5)
 	if err != nil {
 		return answers{}, err
 	}
+	// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
+	out.paused = buildSeatPause(ctx, inv, out.plans)
 
 	// And which machines are not running what the mesh would send them. The same question as a
 	// module being behind its source, one level down: that one says the catalogue is out of date,
@@ -496,7 +538,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
 func (a answers) well() bool {
 	return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
 		len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
-		len(a.filtered) == 0
+		len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
 }
 
 // hostSplit is which machines report which host version, for every version more than one machine

cmd/mesh-controller/supersede_test.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
+// and branch had not built, and closes them as superseded; another repository's plan, another
+// branch's, and a plan made after it are left alone.
+func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
+	at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	sent := at.Add(time.Minute)
+	plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
+		return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
+			Created: created, State: inventory.PlanRolling, Modules: modules}
+	}
+	older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
+		"gitea":    {State: "built", SentAt: &sent}, // done with: stays done
+		"keycloak": {State: "asked"},                // asked, not answered: folded
+		"plex":     {},                              // not yet asked: folded
+		"agent":    {State: "built"},                // built, rolls out, not sent: folded
+		"notes":    {State: "built"},                // built, records: nothing to send
+	})
+	stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
+		"runtime": {State: "asked"},
+	})
+	other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
+	release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
+	later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
+	done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
+	done.State = inventory.PlanDone
+
+	newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
+	newer.Commit = "97b1b2b0c0ffee"
+	rollsOut := func(m string) bool { return m != "notes" }
+	folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
+
+	if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
+		t.Fatalf("folded %v, wanted %v", folded, want)
+	}
+	var ids []string
+	for _, p := range closed {
+		ids = append(ids, p.ID)
+		if p.State != inventory.PlanSuperseded || p.Open() {
+			t.Errorf("%s was left %s", p.ID, p.State)
+		}
+		if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
+			t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
+		}
+	}
+	if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
+		t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
+			"finished one are left alone", ids, want)
+	}
+	if other.State != inventory.PlanRolling {
+		t.Fatal("the plan handed in was changed in place")
+	}
+	if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
+		t.Fatalf("a superseded plan reads %q", line)
+	}
+}
+
+// novox/hq issue 254: a person closes a plan that will not move again, by its id.
+func TestAPersonClosesAStuckPlan(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
+		Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
+	if err := open.inventory.SavePlan(ctx, stuck); err != nil {
+		t.Fatal(err)
+	}
+	if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
+		t.Fatal(err)
+	}
+	closed, err := open.inventory.PlanByID(ctx, stuck.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
+		t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
+	}
+	if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
+		t.Fatal("a plan already closed was closed again")
+	}
+	if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
+		!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
+		t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
+	}
+}

cmd/mesh-controller/unheld_report_test.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
+// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
+// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
+
+func aContainer(name string) catalogue.Manifest {
+	return catalogue.Manifest{Module: name, Version: "1",
+		Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
+}
+
+func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, aContainer("web"))
+	register(t, open, aContainer("db"))
+	// anchor already lacks a runtime for db: true, and not this act's to say.
+	if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil {
+		t.Fatal(err)
+	}
+	said, err := assign(ctx, open, "laptop", "web")
+	if err != nil {
+		t.Fatalf("%v\n%s", err, said)
+	}
+	if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) {
+		t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said)
+	}
+	for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} {
+		if strings.Contains(said, not) {
+			t.Errorf("the assignment says %q, which it did not change:\n%s", not, said)
+		}
+	}
+}
+
+func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
+	shelf := map[string]catalogue.Manifest{
+		"web": aContainer("web"),
+		"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
+			Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
+	}
+	lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
+	if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
+		t.Errorf("meeting a dependency said %v", lines)
+	}
+	// Taking the dependent off says nothing: the dependency went with its module.
+	if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
+		t.Errorf("unassigning the dependent said %v", lines)
+	}
+}
+
+func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
+	unheld := map[string][]catalogue.Unheld{
+		"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
+		"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
+			{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
+	}
+	var named bytes.Buffer
+	reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
+	got := named.String()
+	if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
+		!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
+		t.Errorf("a named push said:\n%s", got)
+	}
+	var all bytes.Buffer
+	reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
+	want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
+	if all.String() != want {
+		t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
+	}
+}
+
+func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
+	read := func(f func()) string {
+		old := os.Stderr
+		r, w, _ := os.Pipe()
+		os.Stderr = w
+		f()
+		_ = w.Close()
+		os.Stderr = old
+		var b bytes.Buffer
+		_, _ = b.ReadFrom(r)
+		return b.String()
+	}
+	u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
+	if got := read(func() { logUnheld("n1", u) }); got != "" {
+		t.Errorf("a command logged:\n%s", got)
+	}
+	logUnheldChanges = true
+	defer func() { logUnheldChanges = false }()
+	if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
+		t.Errorf("the serving controller did not log a change:\n%s", got)
+	}
+	if got := read(func() { logUnheld("n1", u) }); got != "" {
+		t.Errorf("an unchanged report was logged again:\n%s", got)
+	}
+}

cmd/mesh-controller/upgrades.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"path"
 	"regexp"
 	"strings"
 	"time"
@@ -55,10 +56,26 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
 		return nil
 	}
 
+	// **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR
+	// 0218). A merge's plan builds the module and sends it one machine first, the rest once that one
+	// has applied it; this announcement arrives as the build registers, and sending here too — one
+	// machine after another without waiting for any to apply — put the new bundle on every machine in
+	// the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by
+	// hand) is still this handler's.
+	if plans, err := inv.OpenPlans(ctx); err != nil {
+		return notNow(err)
+	} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
+		// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
+		// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
+		fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
+			"`status` lists them as behind and `push --behind` sends it\n",
+			u.Module, shortCommit(u.Commit), id, readableList(on))
+		return nil
+	}
 	if decision.Together {
 		fmt.Printf("%s moved to %s; sending %s together\n",
 			u.Module, shortCommit(u.Commit), readableList(on))
-		return sendTo(ctx, f.open, on)
+		return askAgainOnGrants(sendTo(ctx, f.open, on))
 	}
 	// One at a time, and stopping at the first that fails.
 	//
@@ -69,13 +86,34 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
 		u.Module, shortCommit(u.Commit), readableList(on))
 	for _, node := range on {
 		if err := sendTo(ctx, f.open, []string{node}); err != nil {
-			return fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
+			return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
-				node, u.Module, err)
+				node, u.Module, err))
 		}
 	}
 	return nil
 }
 
+// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
+// an announcement handled by a send whose memberships could not be issued is held and redelivered,
+// rather than taken as handled with the machines left on the old version.
+func askAgainOnGrants(err error) error {
+	if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
+		return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
+	}
+	return err
+}
+
+// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
+// the module that has not finished sending it — or empty when none will (novox/hq issue 249).
+func rolledOutByAPlan(plans []inventory.Plan, module string) string {
+	for _, p := range plans {
+		if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
+			return p.ID
+		}
+	}
+	return ""
+}
+
 // readableList names machines the way a sentence does, because this is read by a person deciding
 // whether an upgrade went where they expected.
 func readableList(names []string) string {
@@ -315,7 +353,53 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
 	for _, e := range moved {
 		movedNames = append(movedNames, e.Manifest.Module)
 	}
+	// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
+	// another controller reading it between the two would ask the tier again.
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return notNow(err)
+	}
+	defer release()
 	plan := planOfMerge(m, movedNames, edges)
+	// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
+	// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
+	// works a repository's modules at a time and a stuck one ends at the next merge.
+	working, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return notNow(err)
+	}
+	rollsOut := func(module string) bool {
+		u, err := inv.UpgradeOf(ctx, module)
+		return err == nil && u.RollOut
+	}
+	folded, superseded := supersededBy(plan, working, rollsOut)
+	if len(folded) > 0 {
+		held := map[string]bool{}
+		for _, e := range entries {
+			held[e.Manifest.Module] = true
+		}
+		names := map[string]bool{}
+		for _, name := range movedNames {
+			names[name] = true
+		}
+		var also []string
+		for _, name := range folded {
+			// One the catalogue no longer holds would fail the newer plan's ask; it is not this
+			// merge's to build.
+			if held[name] && !names[name] {
+				names[name] = true
+				movedNames = append(movedNames, name)
+				also = append(also, name)
+			}
+		}
+		if len(also) > 0 {
+			again := planOfMerge(m, movedNames, edges)
+			again.ID, again.Created = plan.ID, plan.Created
+			plan = again
+			fmt.Printf("  %s, left unbuilt by an older plan of %s, are planned here again\n",
+				strings.Join(also, ", "), plan.Repository)
+		}
+	}
 	if hasCycle(plan.Tiers, edges) {
 		fmt.Printf("  the last tier depends on itself: %s — built together, in no order\n",
 			strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
@@ -323,6 +407,14 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
 	if err := inv.SavePlan(ctx, plan); err != nil {
 		return notNow(err)
 	}
+	// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
+	// both open, which the next merge settles, rather than neither.
+	for _, old := range superseded {
+		if err := inv.SavePlan(ctx, old); err != nil {
+			return notNow(err)
+		}
+		fmt.Printf("  %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
+	}
 	var tiers []string
 	for i, t := range plan.Tiers {
 		tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
@@ -416,25 +508,45 @@ func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
 //
 // A change inside *another* module's directory is that module's business and not this one's, even
 // when the mesh does not hold that module: `known` is every module this repository is known to hold,
-// whatever branch it was registered from. That is also the limit of this — a repository whose shared
+// whatever branch it was registered from.
-// code sits inside a directory the mesh has never seen a module in reads as shared, and everything
+//
-// is rebuilt. Rebuilding too much is the safe direction: the fault this whole path exists for is a
+// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a
-// mesh that believes it is current and is not (novox/hq 04-ISSUES/131).
+// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a
+// change to shared code, because `modules/newmod` was nobody's known directory, and every module
+// built from the repository was rebuilt and rolled out for a module none of them is. So the
+// directories that hold modules are known too: the parents of the known modules' directories
+// (`modules`, never the root). A changed path `<parent>/<name>/…` belongs to the module at
+// `<parent>/<name>` — held or not — and rebuilds nothing else, **provided it is shown to be a
+// module**: its `module.json` is among the changed files (added, changed, or removed with it). A
+// directory under the same parent whose manifest the merge did not touch may as well be a shared
+// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe
+// direction: the fault this whole path exists for is a mesh that believes it is current and is not
+// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was.
 func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
 	// Nothing said about the files, or not all of them said: everything built from it is affected.
 	if len(m.Paths) == 0 || m.PathsTruncated {
 		return candidates
 	}
 	var dirs []string
+	parents := map[string]bool{}
 	for _, e := range known {
 		if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
-			dirs = append(dirs, e.Source.Path)
+			dir := strings.Trim(e.Source.Path, "/")
+			dirs = append(dirs, dir)
+			if parent := path.Dir(dir); parent != "." && parent != "/" {
+				parents[parent] = true
+			}
 		}
 	}
+	changed := map[string]bool{}
 	for _, p := range m.Paths {
-		if !insideAny(p, dirs) {
+		changed[strings.TrimPrefix(p, "/")] = true
-			return candidates
+	}
+	for _, p := range m.Paths {
+		if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) {
+			continue
 		}
+		return candidates
 	}
 	var out []inventory.Entry
 	for _, e := range candidates {
@@ -445,6 +557,28 @@ func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved
 	return out
 }
 
+// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know —
+// `<parent>/<name>/…` under a directory known to hold modules, whose `module.json` the same merge
+// changed (novox/hq issue 252). Such a file is that module's business and nobody else's.
+func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool {
+	p = strings.TrimPrefix(p, "/")
+	for parent := range parents {
+		rest, under := strings.CutPrefix(p, parent+"/")
+		if !under {
+			continue
+		}
+		name, _, inADirectory := strings.Cut(rest, "/")
+		if !inADirectory || name == "" {
+			// A file directly in the parent — `modules/README.md` — is about all of them.
+			continue
+		}
+		if changed[parent+"/"+name+"/module.json"] {
+			return true
+		}
+	}
+	return false
+}
+
 // inside is whether a changed file is in a directory: that directory itself, or under it.
 func inside(path, dir string) bool {
 	dir = strings.Trim(dir, "/")

internal/artifacts/hold.go

@@ -0,0 +1,345 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
+//
+// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
+// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
+// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
+// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
+// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
+// bare blob is unreferenced, so the first real collection would have deleted every archive the
+// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
+// collector runs `--dry-run` until this is true.
+//
+// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
+// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
+// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
+// holder first, which is what lets the bytes go at the next collection.
+//
+// **Nothing a machine reads changes.** The recorded reference stays
+// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
+// before. The holder is the store's bookkeeping, not a second way to reach anything.
+//
+// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
+// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
+// compute which manifest holds any archive from the reference it already has plus one HEAD for the
+// size. No schema change, no second record that could disagree with the store.
+
+const (
+	// mediaManifest is the type a holder is put and asked for as.
+	mediaManifest = "application/vnd.oci.image.manifest.v1+json"
+	// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
+	// whose only purpose is to name its layer.
+	mediaEmpty = "application/vnd.oci.empty.v1+json"
+	// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
+	emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
+	// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
+	// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
+	// not read from anywhere, because the holder must be recomputable from the reference alone.
+	mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
+)
+
+// emptyConfig is the content emptyDigest names.
+var emptyConfig = []byte("{}")
+
+// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
+// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
+// (measured 2026-09-28; internal/builder/registry.go says how that was found).
+var manifestAccept = []string{
+	mediaManifest,
+	"application/vnd.docker.distribution.manifest.v2+json",
+}
+
+type descriptor struct {
+	MediaType string `json:"mediaType"`
+	Digest    string `json:"digest"`
+	Size      int64  `json:"size"`
+}
+
+type holderManifest struct {
+	SchemaVersion int          `json:"schemaVersion"`
+	MediaType     string       `json:"mediaType"`
+	Config        descriptor   `json:"config"`
+	Layers        []descriptor `json:"layers"`
+}
+
+// Holder is the manifest that holds an archive in the store, and its digest.
+//
+// A pure function of the archive's digest and size: the same two in give the same bytes out,
+// always, because `encoding/json` writes a struct's fields in their declared order and there is
+// nothing here that varies by when or where it was composed.
+func Holder(digest string, size int64) (body []byte, holder string) {
+	body, err := json.Marshal(holderManifest{
+		SchemaVersion: 2,
+		MediaType:     mediaManifest,
+		Config:        descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
+		Layers:        []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
+	})
+	if err != nil {
+		// Marshalling a struct of strings and integers cannot fail.
+		panic(err)
+	}
+	sum := sha256.Sum256(body)
+	return body, "sha256:" + hex.EncodeToString(sum[:])
+}
+
+// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
+//
+// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
+// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
+// is what lets the sweep run it over every kept archive on every build and so backfill the bare
+// blobs published before holders existed (novox/hq issue 253).
+//
+// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
+// fact the caller reports rather than one this invents a remedy for.
+func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
+	repository, digest, archive, err := s.archive(reference)
+	if err != nil || !archive {
+		return false, err
+	}
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return false, err
+	}
+	return s.HoldBlob(ctx, repository, digest, size)
+}
+
+// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
+// question an operator needs answered with "none unheld" before the collector is let loose.
+//
+// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
+func (s Store) Held(ctx context.Context, reference string) (bool, error) {
+	repository, digest, archive, err := s.archive(reference)
+	if err != nil {
+		return false, err
+	}
+	if !archive {
+		return true, nil
+	}
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return false, err
+	}
+	_, holder := Holder(digest, size)
+	return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
+}
+
+// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
+// there already. Answers whether it wrote one.
+//
+// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
+// the store reports. Both arrive at the same holder, which is the point of composing it.
+func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
+	if s.Address == "" {
+		return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
+	}
+	body, holder := Holder(digest, size)
+	there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
+	if err != nil {
+		return false, err
+	}
+	if there {
+		return false, nil
+	}
+
+	// The config must be in the repository before a manifest naming it is accepted: a registry
+	// refuses a manifest whose blobs it cannot find there, which is the property that makes a
+	// holder mean something.
+	if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
+		return false, err
+	}
+
+	// **By digest, never by tag.** A tag would be one more name to move and one more thing the
+	// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
+	// it pins by digest, and an untagged manifest is kept by plain collection.
+	request, err := http.NewRequestWithContext(ctx, http.MethodPut,
+		s.url(repository, "manifests", holder), bytes.NewReader(body))
+	if err != nil {
+		return false, err
+	}
+	request.Header.Set("Content-Type", mediaManifest)
+	response, err := s.client().Do(request)
+	if err != nil {
+		return false, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusCreated {
+		said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
+		return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
+			repository, digest, response.Status, strings.TrimSpace(string(said)))
+	}
+	return true, nil
+}
+
+// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
+//
+// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
+// collector would keep its bytes for ever on the strength of it — the sweep would record the
+// archive collected while the disk said otherwise. Deleting the holder first and failing before
+// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
+//
+// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
+// and without the blob there is nothing left for a holder to keep. A store that never had a holder
+// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
+// that is the outcome wanted.
+func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return err
+	}
+	_, holder := Holder(digest, size)
+	err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
+	if err == Gone {
+		return nil
+	}
+	return err
+}
+
+// archive reads a recorded reference into its repository and digest, and whether it is an archive
+// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
+func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
+	path, kept := catalogue.InArtifactStore(reference)
+	if !kept {
+		return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
+	}
+	if s.Address == "" {
+		return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
+	}
+	repository, kind, digest, err := split(path)
+	if err != nil {
+		return "", "", false, err
+	}
+	return repository, digest, kind == "blobs", nil
+}
+
+// blobSize is how large the store says a blob is; Gone when it does not have it.
+func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
+	if err != nil {
+		return 0, err
+	}
+	response, err := s.client().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
+	}
+	defer response.Body.Close()
+	switch response.StatusCode {
+	case http.StatusOK:
+	case http.StatusNotFound:
+		return 0, Gone
+	default:
+		return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
+	}
+	// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
+	// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
+	// it, and the header is what the registry itself wrote.
+	if length := response.Header.Get("Content-Length"); length != "" {
+		if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
+			return n, nil
+		}
+	}
+	if response.ContentLength >= 0 {
+		return response.ContentLength, nil
+	}
+	return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
+}
+
+// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
+// naming the digest — the registry's own two steps, the same the builder takes for an archive.
+func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
+	if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
+		return err
+	} else if there {
+		return nil
+	}
+	start, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
+	if err != nil {
+		return err
+	}
+	begun, err := s.client().Do(start)
+	if err != nil {
+		return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
+	}
+	begun.Body.Close()
+	if begun.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
+	}
+	where := begun.Header.Get("Location")
+	if where == "" {
+		return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
+	}
+	if strings.HasPrefix(where, "/") {
+		where = "http://" + s.Address + where
+	}
+	separator := "?"
+	if strings.Contains(where, "?") {
+		separator = "&"
+	}
+	put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	put.Header.Set("Content-Type", "application/octet-stream")
+	done, err := s.client().Do(put)
+	if err != nil {
+		return err
+	}
+	defer done.Body.Close()
+	if done.StatusCode != http.StatusCreated {
+		said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
+		return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
+	}
+	return nil
+}
+
+// has is whether the store answers 200 for a HEAD at that URL.
+func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
+	if err != nil {
+		return false, err
+	}
+	for _, media := range accept {
+		request.Header.Add("Accept", media)
+	}
+	response, err := s.client().Do(request)
+	if err != nil {
+		return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
+	}
+	defer response.Body.Close()
+	switch response.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusNotFound:
+		return false, nil
+	default:
+		return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
+	}
+}
+
+func (s Store) url(repository, kind, digest string) string {
+	return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
+}
+
+func (s Store) client() *http.Client {
+	if s.HTTP != nil {
+		return s.HTTP
+	}
+	return &http.Client{Timeout: 30 * time.Second}
+}

internal/artifacts/hold_test.go

@@ -0,0 +1,268 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
+//
+// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
+// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
+// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
+// type. What is asserted is this side's decisions; the live test below asserts the registry's.
+
+type memRegistry struct {
+	mu        sync.Mutex
+	blobs     map[string][]byte // repository + "@" + digest
+	manifests map[string][]byte // repository + "@" + digest
+	writes    []string          // every PUT and DELETE, as "METHOD path"
+}
+
+func digestOf(body []byte) string {
+	sum := sha256.Sum256(body)
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+func (m *memRegistry) serve(t *testing.T) Store {
+	t.Helper()
+	m.blobs = map[string][]byte{}
+	m.manifests = map[string][]byte{}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		path := strings.TrimPrefix(r.URL.Path, "/v2/")
+		if r.Method == http.MethodPut || r.Method == http.MethodDelete {
+			m.writes = append(m.writes, r.Method+" "+r.URL.Path)
+		}
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
+			repository := strings.TrimSuffix(path, "/blobs/uploads/")
+			w.Header().Set("Location", "/upload/"+repository+"?state=x")
+			w.WriteHeader(http.StatusAccepted)
+		case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
+			repository := strings.TrimPrefix(r.URL.Path, "/upload/")
+			body, _ := io.ReadAll(r.Body)
+			digest := r.URL.Query().Get("digest")
+			if digest != digestOf(body) {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			m.blobs[repository+"@"+digest] = body
+			w.WriteHeader(http.StatusCreated)
+		case strings.Contains(path, "/blobs/"):
+			repository, digest, _ := strings.Cut(path, "/blobs/")
+			key := repository + "@" + digest
+			body, ok := m.blobs[key]
+			if !ok {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			switch r.Method {
+			case http.MethodHead:
+				w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+				w.WriteHeader(http.StatusOK)
+			case http.MethodDelete:
+				delete(m.blobs, key)
+				w.WriteHeader(http.StatusAccepted)
+			default:
+				w.WriteHeader(http.StatusMethodNotAllowed)
+			}
+		case strings.Contains(path, "/manifests/"):
+			repository, digest, _ := strings.Cut(path, "/manifests/")
+			key := repository + "@" + digest
+			switch r.Method {
+			case http.MethodHead:
+				if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			case http.MethodPut:
+				body, _ := io.ReadAll(r.Body)
+				if digest != digestOf(body) {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				var named holderManifest
+				if err := json.Unmarshal(body, &named); err != nil {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				for _, d := range append([]descriptor{named.Config}, named.Layers...) {
+					if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
+						w.WriteHeader(http.StatusBadRequest)
+						fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
+						return
+					}
+				}
+				m.manifests[key] = body
+				w.WriteHeader(http.StatusCreated)
+			case http.MethodDelete:
+				if _, ok := m.manifests[key]; !ok {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+				delete(m.manifests, key)
+				w.WriteHeader(http.StatusAccepted)
+			}
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	return Store{Address: strings.TrimPrefix(server.URL, "http://")}
+}
+
+// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
+func (m *memRegistry) bare(repository string, body []byte) string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	digest := digestOf(body)
+	m.blobs[repository+"@"+digest] = body
+	return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
+}
+
+func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
+	// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
+	// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
+	// never be mistaken for one of a different blob.
+	digest := "sha256:" + strings.Repeat("a", 64)
+	one, first := Holder(digest, 42)
+	two, second := Holder(digest, 42)
+	if !bytes.Equal(one, two) || first != second {
+		t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
+	}
+	if _, other := Holder(digest, 43); other == first {
+		t.Fatal("a different size composed the same holder")
+	}
+	want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
+		`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
+		`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
+		`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
+	if string(one) != want {
+		t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
+	}
+	if digestOf(emptyConfig) != emptyDigest {
+		t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
+	}
+}
+
+func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
+	// The archives published before this have no holder. Hold, run over every kept archive on
+	// every sweep, writes one the first time and nothing after.
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+	body := []byte("a theme")
+	reference := m.bare("shell/config", body)
+
+	if held, err := store.Held(ctx, reference); err != nil || held {
+		t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
+	}
+	wrote, err := store.Hold(ctx, reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !wrote {
+		t.Fatal("holding a bare archive wrote nothing")
+	}
+	_, holder := Holder(digestOf(body), int64(len(body)))
+	if _, ok := m.manifests["shell/config@"+holder]; !ok {
+		t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
+	}
+	if held, err := store.Held(ctx, reference); err != nil || !held {
+		t.Fatalf("after holding, held=%v (%v)", held, err)
+	}
+
+	writes := len(m.writes)
+	wrote, err = store.Hold(ctx, reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wrote || len(m.writes) != writes {
+		t.Fatalf("holding again wrote %v", m.writes[writes:])
+	}
+}
+
+func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+
+	// An image is its own manifest: nothing is asked.
+	wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
+	if err != nil || wrote || len(m.writes) != 0 {
+		t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
+	}
+	// An archive the store does not have is a fact to report, not something to invent a holder for.
+	_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
+	if !errors.Is(err, Gone) {
+		t.Fatalf("holding a missing archive answered %v, want Gone", err)
+	}
+	// And a reference that is not the mesh's is refused as such.
+	if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
+		t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
+	}
+}
+
+func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
+	// A holder left behind would keep the bytes through every collection while the record said
+	// collected; the link deleted first and the holder failing after would be that exactly.
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+	body := []byte("an old theme")
+	reference := m.bare("shell/config", body)
+	if _, err := store.Hold(ctx, reference); err != nil {
+		t.Fatal(err)
+	}
+	m.writes = nil
+
+	if err := store.LetGo(ctx, reference); err != nil {
+		t.Fatal(err)
+	}
+	_, holder := Holder(digestOf(body), int64(len(body)))
+	want := []string{
+		"DELETE /v2/shell/config/manifests/" + holder,
+		"DELETE /v2/shell/config/blobs/" + digestOf(body),
+	}
+	if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
+		t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
+	}
+	if len(m.manifests) != 0 {
+		t.Fatalf("a holder survived: %v", m.manifests)
+	}
+	// Asked again, the archive is already gone, which is the outcome wanted.
+	if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
+		t.Fatalf("letting go twice answered %v, want Gone", err)
+	}
+}
+
+func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
+	// An archive published before holders and let go of before any sweep held it: the holder's
+	// delete answers 404, which is the outcome wanted, and the blob still goes.
+	m := &memRegistry{}
+	store := m.serve(t)
+	reference := m.bare("shell/config", []byte("never held"))
+	if err := store.LetGo(context.Background(), reference); err != nil {
+		t.Fatal(err)
+	}
+	if len(m.blobs) != 0 {
+		t.Fatalf("the blob survived: %v", m.blobs)
+	}
+}

internal/artifacts/live_registry_test.go

@@ -0,0 +1,130 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The registry's own collector keeps a held archive and takes a bare one (novox/hq issue 253,
+// ADR 0189).
+//
+// Everything else here is asserted against a fake, which can only say what this side asks. This is
+// the one question a fake cannot answer — what `registry garbage-collect` actually does with what
+// this side wrote — and it is the whole of whether the store's nightly step may stop being a dry
+// run. Against the very image the mesh's store runs:
+//
+//	docker run -d --rm --name mesh-controller-registry -p 15000:5000 \
+//	    -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2.8.3
+//	MESH_TEST_REGISTRY=127.0.0.1:15000 MESH_TEST_REGISTRY_CONTAINER=mesh-controller-registry \
+//	    go test -run Live ./internal/artifacts/
+//	docker stop mesh-controller-registry
+//
+// Skipped without both variables: it needs a registry it may write to and collect, and a container
+// to run the collector in.
+func TestLiveTheRegistrysCollectorKeepsWhatIsHeldAndTakesWhatIsNot(t *testing.T) {
+	address := os.Getenv("MESH_TEST_REGISTRY")
+	container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER")
+	if address == "" || container == "" {
+		t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see this test's comment for the registry to raise")
+	}
+	ctx := context.Background()
+	store := Store{Address: address}
+	run := time.Now().UnixNano()
+
+	// Four archives in four repositories, each a different story. Distinct bytes per run, so a
+	// registry reused across runs cannot answer for an earlier one.
+	put := func(name string) (repository, digest string, body []byte) {
+		repository = fmt.Sprintf("live-%d/%s", run, name)
+		body = []byte(fmt.Sprintf("%s archive of run %d", name, run))
+		digest = digestOf(body)
+		if err := store.putBlob(ctx, repository, digest, body); err != nil {
+			t.Fatal(err)
+		}
+		return repository, digest, body
+	}
+	reference := func(repository, digest string) string {
+		return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
+	}
+
+	// Published held — what PublishArchive now does.
+	heldRepo, heldDigest, heldBody := put("held")
+	if _, err := store.HoldBlob(ctx, heldRepo, heldDigest, int64(len(heldBody))); err != nil {
+		t.Fatalf("the registry refused a holder: %v", err)
+	}
+	// Published bare, as before, and never held: what the collector must take.
+	_, bareDigest, _ := put("bare")
+	// Published bare and then held by the sweep: the backfill.
+	backRepo, backDigest, backBody := put("backfilled")
+	if wrote, err := store.Hold(ctx, reference(backRepo, backDigest)); err != nil || !wrote {
+		t.Fatalf("backfilling wrote=%v: %v", wrote, err)
+	}
+	if held, err := store.Held(ctx, reference(backRepo, backDigest)); err != nil || !held {
+		t.Fatalf("after backfilling, held=%v: %v", held, err)
+	}
+	// Held, and then let go of by the sweep: holder first, then the link.
+	goneRepo, goneDigest, _ := put("let-go")
+	if _, err := store.Hold(ctx, reference(goneRepo, goneDigest)); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.LetGo(ctx, reference(goneRepo, goneDigest)); err != nil {
+		t.Fatalf("letting go of a held archive: %v", err)
+	}
+
+	collected, err := exec.CommandContext(ctx, "docker", "exec", container,
+		"registry", "garbage-collect", "/etc/docker/registry/config.yml").CombinedOutput()
+	if err != nil {
+		t.Fatalf("the collector failed: %v\n%s", err, collected)
+	}
+	t.Logf("the collector said:\n%s", lastLines(string(collected), 12))
+
+	// What is asserted is the bytes on the store's disk, not what the running server answers: the
+	// server caches blob descriptors in memory and can answer for a blob the collector removed.
+	onDisk := func(digest string) bool {
+		hex := strings.TrimPrefix(digest, "sha256:")
+		path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data"
+		return exec.CommandContext(ctx, "docker", "exec", container, "test", "-f", path).Run() == nil
+	}
+	if !onDisk(heldDigest) {
+		t.Error("the collector took an archive published held")
+	}
+	if !onDisk(backDigest) {
+		t.Error("the collector took an archive the sweep backfilled a holder for")
+	}
+	if onDisk(bareDigest) {
+		t.Error("the collector kept a bare archive — then the holders prove nothing, and this test is wrong")
+	}
+	if onDisk(goneDigest) {
+		t.Error("the collector kept an archive the sweep let go of: its holder outlived its link")
+	}
+	// And what survived is still fetched exactly as machines fetch it: the blob, by digest.
+	for repository, want := range map[string][]byte{heldRepo: heldBody, backRepo: backBody} {
+		digest := digestOf(want)
+		response, err := http.Get(catalogue.Routed(reference(repository, digest), address))
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, _ := io.ReadAll(response.Body)
+		response.Body.Close()
+		if response.StatusCode != http.StatusOK || !bytes.Equal(got, want) {
+			t.Errorf("%s answered %s with %q after collection", repository, response.Status, got)
+		}
+	}
+}
+
+func lastLines(s string, n int) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}

internal/artifacts/store.go

@@ -0,0 +1,123 @@
+// Package artifacts speaks to the mesh's artifact store over its own door.
+//
+// Only what the mesh needs that nothing else does: letting go of something it put there
+// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
+// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
+// is every machine's, through its runtime. This is the one operation that belongs to the thing
+// holding the records, because it is the only one that is a decision rather than a transfer.
+package artifacts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Store is the artifact store at an address, as this machine reaches it.
+type Store struct {
+	// Address is `host:port` — the store as the caller reaches it now, composed and never
+	// recorded (novox/hq 04-ISSUES/102).
+	Address string
+	// HTTP is the client used; nil is a client with a modest timeout.
+	HTTP *http.Client
+}
+
+// Gone is the answer when the store does not hold it: the outcome wanted, already true.
+var Gone = errors.New("the store does not hold it")
+
+// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing
+// the store holds by digest.
+//
+// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite
+// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an
+// error" is how a cautious loop became one that did nothing while reporting the right number.
+var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
+
+// LetGo asks the store to drop one artifact the mesh recorded making.
+//
+// Takes a reference as the mesh records it — `artifact-store://<module>/<artifact>@sha256:…` for
+// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
+// and composes the address here at the moment of use.
+//
+// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
+// issue 253); an image in one.
+//
+// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
+// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
+// which of the two happened.
+func (s Store) LetGo(ctx context.Context, reference string) error {
+	// **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its
+	// own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…`
+	// as the mesh's too — it cannot tell one registry host from another — so normalising belongs
+	// where the provenance is known, which is the sweep reading its own build records, not here
+	// where the only job is to refuse anything that is not plainly ours.
+	path, kept := catalogue.InArtifactStore(reference)
+	if !kept {
+		// Nothing the mesh put in its own store. Refused rather than attempted: composing a
+		// delete for a reference of unknown shape is how a sweep reaches something that is not
+		// the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on.
+		return fmt.Errorf("%w: %s", ErrNotOurs, reference)
+	}
+	if s.Address == "" {
+		return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
+	}
+	repository, kind, digest, err := split(path)
+	if err != nil {
+		return err
+	}
+	if kind == "blobs" {
+		// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
+		// naming the blob would keep its bytes through every collection while the record said
+		// collected. Gone here means the store has no such blob, so there is nothing to let go.
+		if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
+			return err
+		}
+	}
+	return s.remove(ctx, s.url(repository, kind, digest), reference)
+}
+
+// remove asks the store to delete what is at url. Gone when it has no such thing.
+func (s Store) remove(ctx context.Context, url, what string) error {
+	request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	response, err := s.client().Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	switch response.StatusCode {
+	case http.StatusAccepted, http.StatusOK, http.StatusNoContent:
+		return nil
+	case http.StatusNotFound:
+		return Gone
+	case http.StatusMethodNotAllowed:
+		// The registry was started without deletion enabled. Said plainly, because the remedy is
+		// a setting on the store's module and not anything about this artifact.
+		return fmt.Errorf(
+			"the artifact store refuses deletion: its server was started without it enabled "+
+				"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
+				"module is applied again (novox/hq ADR 0189). Asking about %s", what)
+	default:
+		return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
+	}
+}
+
+// split reads a recorded path into the repository, which endpoint names the thing, and the digest.
+//
+// Two shapes, which are the two the mesh records: `<repository>@sha256:<hex>` is a manifest, and
+// `<repository>/blobs/sha256:<hex>` is a blob.
+func split(path string) (repository, kind, digest string, err error) {
+	if before, after, ok := strings.Cut(path, "@sha256:"); ok {
+		return before, "manifests", "sha256:" + after, nil
+	}
+	if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok {
+		return before, "blobs", "sha256:" + after, nil
+	}
+	return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path)
+}

internal/artifacts/store_test.go

@@ -0,0 +1,134 @@
+package artifacts
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
+//
+// A fake store records what it was asked to delete, so what is asserted is the mesh's decision
+// and the shape of the request — not the registry's behaviour, which is the registry's to test.
+
+func fakeStore(t *testing.T, answer int) (Store, *[]string) {
+	t.Helper()
+	var asked []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
+			// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
+			// that does not have the thing does not have its blob either.
+			if answer == http.StatusNotFound {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			w.Header().Set("Content-Length", "7")
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		if r.Method != http.MethodDelete {
+			t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
+		}
+		asked = append(asked, r.URL.Path)
+		w.WriteHeader(answer)
+	}))
+	t.Cleanup(server.Close)
+	return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked
+}
+
+func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
+	// The two shapes the mesh records: a manifest by digest, and a blob by digest. They are
+	// different endpoints, and asking at the wrong one answers 404 — which this would then
+	// record as collected, leaving the bytes on disk for ever while the record says otherwise.
+	store, asked := fakeStore(t, http.StatusAccepted)
+	ctx := context.Background()
+
+	image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123"
+	archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456"
+	if err := store.LetGo(ctx, image); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.LetGo(ctx, archive); err != nil {
+		t.Fatal(err)
+	}
+	// The archive's holder goes first, then the archive (novox/hq issue 253).
+	_, holder := Holder("sha256:def456", 7)
+	want := []string{
+		"/v2/web/app/manifests/sha256:abc123",
+		"/v2/web/config/manifests/" + holder,
+		"/v2/web/config/blobs/sha256:def456",
+	}
+	if strings.Join(*asked, " ") != strings.Join(want, " ") {
+		t.Fatalf("the store was asked %v; want %v", *asked, want)
+	}
+}
+
+func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) {
+	// The outcome wanted, already true. Told apart from success only so the sweep can say which
+	// happened; both are recorded, because retrying for ever is the thing to avoid.
+	store, _ := fakeStore(t, http.StatusNotFound)
+	err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
+	if !errors.Is(err, Gone) {
+		t.Fatalf("a store that does not hold it answered %v, want Gone", err)
+	}
+}
+
+func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) {
+	// The registry answers 405 when it was started without deletion enabled. The remedy is a
+	// setting on the store's module, and saying "405" would send somebody to the wrong place.
+	store, _ := fakeStore(t, http.StatusMethodNotAllowed)
+	err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123")
+	if err == nil {
+		t.Fatal("a store that refuses deletion was read as success")
+	}
+	if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") {
+		t.Fatalf("the refusal does not name the remedy: %v", err)
+	}
+}
+
+func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) {
+	// The whole safety of the sweep is that it names only what the mesh recorded putting there.
+	// A reference of another shape — a vendor's image, a package version — is refused rather
+	// than composed into a delete somewhere that is not the mesh's store.
+	store, asked := fakeStore(t, http.StatusAccepted)
+	for _, reference := range []string{
+		"docker.io/library/registry@sha256:abc123",
+		"registry@sha256:abc123",
+		"1.4.2",
+	} {
+		if err := store.LetGo(context.Background(), reference); err == nil {
+			t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference)
+		}
+	}
+	if len(*asked) != 0 {
+		t.Fatalf("the store was asked about %v", *asked)
+	}
+}
+
+// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the
+// record and not about the store (novox/hq issue 226).
+//
+// The sweep skips one and abandons itself for the other, so they cannot be the same error. The
+// first live run met a reference recorded with the store's old address, read the refusal as "the
+// store refuses everything", and collected none of the 1681 it had found.
+func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) {
+	store, asked := fakeStore(t, http.StatusAccepted)
+	for _, reference := range []string{
+		"docker.io/library/registry@sha256:abc123",
+		"127.0.0.1:5100/mesh-tools/build@sha256:abc123",
+		"1.4.2",
+	} {
+		err := store.LetGo(context.Background(), reference)
+		if !errors.Is(err, ErrNotOurs) {
+			t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err)
+		}
+	}
+	if len(*asked) != 0 {
+		t.Fatalf("the store was asked about %v", *asked)
+	}
+}

internal/broker/cancelled.go

@@ -0,0 +1,92 @@
+package broker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A seat's cancelled set (novox/hq ADR 0219).
+//
+// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
+// race no ordering on one side closes: a holder may fetch the ask in the moment between the
+// controller reading the queue and deleting the message, and build what a person cancelled. So the
+// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
+// there after taking it and before building: listed, it terminates the ask and announces it failed
+// rather than starting it.
+//
+// **A key-value bucket, because that is the shape the bus already has for "a small set the
+// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
+// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
+// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
+// three objects of one work queue read as one family; asserted by the controller with the queue,
+// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
+
+// CancelledSetName is the bucket holding a seat's cancelled asks.
+func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
+
+// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
+// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
+func hasCancelledSet(seat string) bool {
+	for _, s := range seatsTheControllerAsks {
+		if s == seat {
+			return true
+		}
+	}
+	return false
+}
+
+// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
+// own, and never one to report as state nothing declares.
+func IsCancelledSet(bucket string) bool {
+	return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
+}
+
+// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
+const cancelledSetAge = 7 * 24 * time.Hour
+
+// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
+type CancelledSetAsserter interface {
+	EnsureCancelledSet(seat string) error
+}
+
+// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
+func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
+	for _, s := range seats {
+		if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
+			continue
+		}
+		if err := a.EnsureCancelledSet(s.Name); err != nil {
+			return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
+		}
+	}
+	return nil
+}
+
+// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
+// Direct reads on, which is how a holder looks an id up with one request.
+func (j *JetStream) EnsureCancelledSet(seat string) error {
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket: CancelledSetName(seat),
+		Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
+			"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
+			"so an ask fetched in that moment is ended rather than built", seat),
+		History:      1,
+		TTL:          cancelledSetAge,
+		MaxValueSize: 4 * 1024,
+		MaxBytes:     4 * 1024 * 1024,
+		Storage:      jetstream.FileStorage,
+	}); err != nil {
+		return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
+	}
+	return nil
+}

internal/broker/cancelled_test.go

@@ -0,0 +1,73 @@
+package broker
+
+import "testing"
+
+// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own
+// machine's subjects, and says whether it is paused under its own machine's name; the controller may
+// ask any machine's holder its verbs (novox/hq ADR 0219).
+func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) {
+	seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"},
+		Emits:  []string{"started", "built", "log.*", "paused.*"},
+		Serves: []string{"current", "kill", "pause", "resume"}}
+	holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent",
+		Holds: []Seat{seat}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
+	hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
+	has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.ace")
+	hasNot(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*")
+	has(t, holder.Publish, "mesh.seat.node-build-agent.event.log.*")
+	has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace")
+	hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14")
+
+	// A module's own seat's queue is its own affair: no cancelled set, no grant for one.
+	other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram",
+		Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"})
+	hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>")
+
+	controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>")
+
+	if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" ||
+		!IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") {
+		t.Error("the cancelled set is not named as its seat's family")
+	}
+}
+
+// Only the work queues the controller asks get a cancelled set.
+func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) {
+	var asserted []string
+	a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil })
+	if err := RaiseCancelledSets(a, []DeclaredSeat{
+		{Name: "node-build-agent", Accepts: []string{"build"}},
+		{Name: "telegram-sender", Accepts: []string{"send"}},
+		{Name: "mesh-controller"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if len(asserted) != 1 || asserted[0] != "node-build-agent" {
+		t.Fatalf("asserted %v", asserted)
+	}
+}
+
+type asserterFunc func(string) error
+
+func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) }
+
+// A seat's cancelled set is never reported as state nothing declares.
+func TestACancelledSetIsNotUndeclaredState(t *testing.T) {
+	undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(undeclared) != 1 || undeclared[0] != "gone_state" {
+		t.Fatalf("undeclared %v", undeclared)
+	}
+}
+
+type fakeBuckets struct{ names []string }
+
+func (f fakeBuckets) EnsureBucket(Bucket) error      { return nil }
+func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil }

internal/broker/consumer_from_now_test.go

@@ -0,0 +1,91 @@
+package broker
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// A consumer that reacts to announcements, made on a stream that keeps a week of them, starts from now:
+// made from the start it replays every merge and build of that week (novox/hq issue 248). And a reset
+// re-makes a stuck one from now, its configuration otherwise kept, and refuses a work queue.
+//
+//	docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
+//	MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAConsumerMadeFromNow
+func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	js, err := Dial(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	const stream = "HISTORY_TEST"
+	_ = js.js.DeleteStream(stream)
+	if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{"history.>"}, Storage: nats.MemoryStorage}); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.js.DeleteStream(stream) }()
+	for i := 0; i < 5; i++ {
+		if _, err := js.js.Publish("history.merged", []byte(fmt.Sprint(i))); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	fresh := Consumer{Name: "fresh", Stream: stream, Filters: []string{"history.merged"}, Push: true,
+		AckWaitSeconds: 30, MaxDeliver: 5, MaxAckPending: 1, FromNow: true}
+	if err := js.EnsureConsumer(fresh); err != nil {
+		t.Fatal(err)
+	}
+	info, _ := js.js.ConsumerInfo(stream, "fresh")
+	if info.NumPending != 0 || info.Config.DeliverPolicy != nats.DeliverNewPolicy {
+		t.Fatalf("a consumer made from now holds %d of the stream's past (policy %v)", info.NumPending, info.Config.DeliverPolicy)
+	}
+	_, _ = js.js.Publish("history.merged", []byte("new"))
+	time.Sleep(100 * time.Millisecond)
+	if info, _ = js.js.ConsumerInfo(stream, "fresh"); info.NumPending != 1 {
+		t.Fatalf("a new announcement is not pending: %d", info.NumPending)
+	}
+	// Asserted again, it keeps where it is.
+	if err := js.EnsureConsumer(fresh); err != nil {
+		t.Fatal(err)
+	}
+
+	// One made from the start, as the server's default makes it, and stuck behind its history.
+	old := fresh
+	old.Name, old.FromNow = "old", false
+	if err := js.EnsureConsumer(old); err != nil {
+		t.Fatal(err)
+	}
+	if info, _ = js.js.ConsumerInfo(stream, "old"); info.NumPending != 6 {
+		t.Fatalf("the default should replay all six: %d", info.NumPending)
+	}
+	before, after, err := js.ResetConsumer(stream, "old")
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, _ = js.js.ConsumerInfo(stream, "old")
+	if before.Pending != 6 || after.Pending != 0 || info.Config.MaxAckPending != 1 || info.Config.MaxDeliver != 5 ||
+		info.Config.AckWait != 30*time.Second || info.Config.DeliverSubject == "" {
+		t.Fatalf("before %+v after %+v config %+v", before, after, info.Config)
+	}
+
+	// Never a work queue: what is pending there is work.
+	const queue = "QUEUE_TEST"
+	_ = js.js.DeleteStream(queue)
+	if _, err := js.js.AddStream(&nats.StreamConfig{Name: queue, Subjects: []string{"queue.>"}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage}); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.js.DeleteStream(queue) }()
+	if _, err := js.js.AddConsumer(queue, &nats.ConsumerConfig{Durable: "w", AckPolicy: nats.AckExplicitPolicy}); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := js.ResetConsumer(queue, "w"); err == nil {
+		t.Fatal("a work queue's consumer was reset")
+	}
+}

internal/broker/derived.go

@@ -47,7 +47,13 @@ type Consumer struct {
 	// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
 	// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
 	MaxAckPending int
-	Why           string
+	// FromNow makes a consumer that does not exist yet start at the stream's end rather than its
+	// beginning (novox/hq issue 248). For a consumer that reacts to announcements — a merge, a build's
+	// outcome — on a stream that keeps a week of them: the server's default, everything the stream
+	// holds, replays every merge and every build of that week as if it had just happened. A consumer
+	// that exists keeps where it is, whatever this says; only its making is decided here.
+	FromNow bool
+	Why     string
 }
 
 // seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
@@ -144,6 +150,44 @@ func ConsumerFor(p Principal) (Consumer, bool) {
 	}, true
 }
 
+// ModuleConsumer is one module's durable consumer, with the module and node it is for.
+type ModuleConsumer struct {
+	Node, Module string
+	Consumer     Consumer
+}
+
+// ConsumersOf is every module's durable consumer the composed users imply: each module user's, and
+// each module a runtime carries — a carried module with no account of its own is no user (novox/hq
+// issue 195), and its consumer is still the controller's to make, since the runtime reads it on the
+// module's behalf (ADR 0198). One per module and node, whichever of the two named it first.
+func ConsumersOf(users []Principal) []ModuleConsumer {
+	var out []ModuleConsumer
+	seen := map[string]bool{}
+	add := func(p Principal) {
+		c, needed := ConsumerFor(p)
+		if !needed || seen[p.Node+"/"+p.Module] {
+			return
+		}
+		seen[p.Node+"/"+p.Module] = true
+		out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
+	}
+	for _, p := range users {
+		if p.Kind == KindModule {
+			add(p)
+		}
+	}
+	for _, p := range users {
+		if p.Kind != KindNodeTools {
+			continue
+		}
+		for _, d := range p.Carries {
+			add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
+				Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches})
+		}
+	}
+	return out
+}
+
 // HolderConsumerFor is the worker a seat's holders share on that seat's work queue.
 //
 // **One worker for every holder, and each holder pulls one ask when it is idle** (novox/hq ADR

internal/broker/jetstream.go

@@ -1,6 +1,7 @@
 package broker
 
 import (
+	"context"
 	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
@@ -12,6 +13,7 @@ import (
 	"time"
 
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 )
 
 // The JetStream side of the controller: the one place the mesh's streams and consumers are
@@ -84,6 +86,13 @@ func pinnedTo(path string) (*tls.Config, error) {
 	return PinnedToFingerprint(want), nil
 }
 
+// OnConn is the JetStream handle over a connection the caller already holds — the control plane's
+// link — for asserting what the bus holds without dialling a second time.
+func OnConn(conn *nats.Conn) *JetStream {
+	js, _ := conn.JetStream()
+	return &JetStream{conn: conn, js: js}
+}
+
 // DialPinned is Dial with the server's certificate pinned by a fingerprint the caller already holds
 // — a module or a build machine that was handed one beside its credential, and has no file.
 func DialPinned(url, fingerprint string, opts ...nats.Option) (*JetStream, error) {
@@ -299,6 +308,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
 		}
 		return nil
 	case errors.Is(err, nats.ErrConsumerNotFound):
+		if c.FromNow {
+			want.DeliverPolicy = nats.DeliverNewPolicy
+		}
 		if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
 			return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
 		}
@@ -308,6 +320,51 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
 	}
 }
 
+// ConsumerState is what a reset says about a consumer, before and after.
+type ConsumerState struct {
+	DeliverPolicy string
+	Delivered     uint64
+	AckFloor      uint64
+	Pending       uint64
+	AckPending    int
+}
+
+func stateOf(info *nats.ConsumerInfo) ConsumerState {
+	policy, _ := info.Config.DeliverPolicy.MarshalJSON()
+	return ConsumerState{DeliverPolicy: strings.Trim(string(policy), `"`), Delivered: info.Delivered.Stream,
+		AckFloor: info.AckFloor.Stream, Pending: info.NumPending, AckPending: info.NumAckPending}
+}
+
+// ResetConsumer re-makes a consumer to start from now, its configuration otherwise unchanged (novox/hq
+// issue 248): what it had not yet delivered or acknowledged is dropped, which is the point — on a stream
+// that keeps history, a consumer replaying a week of announcements does nothing anyone wants. Refused on
+// a work queue, where what is pending is work nobody else will do.
+func (j *JetStream) ResetConsumer(stream, name string) (before, after ConsumerState, err error) {
+	info, err := j.js.StreamInfo(stream)
+	if err != nil {
+		return before, after, fmt.Errorf("asking about stream %s: %w", stream, err)
+	}
+	if info.Config.Retention == nats.WorkQueuePolicy {
+		return before, after, fmt.Errorf("%s is a work queue: what its consumer has pending is work, and a reset would drop it", stream)
+	}
+	have, err := j.js.ConsumerInfo(stream, name)
+	if err != nil {
+		return before, after, fmt.Errorf("asking about consumer %s on %s: %w", name, stream, err)
+	}
+	before = stateOf(have)
+	want := have.Config
+	want.DeliverPolicy = nats.DeliverNewPolicy
+	want.OptStartSeq, want.OptStartTime = 0, nil
+	if err := j.js.DeleteConsumer(stream, name); err != nil {
+		return before, after, fmt.Errorf("removing consumer %s on %s: %w", name, stream, err)
+	}
+	made, err := j.js.AddConsumer(stream, &want)
+	if err != nil {
+		return before, after, fmt.Errorf("re-making consumer %s on %s — it is gone until the controller asserts it at its next start: %w", name, stream, err)
+	}
+	return before, stateOf(made), nil
+}
+
 func retentionOf(r Retention) nats.RetentionPolicy {
 	switch r {
 	case RetentionWorkQueue:
@@ -316,3 +373,50 @@ func retentionOf(r Retention) nats.RetentionPolicy {
 		return nats.LimitsPolicy
 	}
 }
+
+// EnsureBucket creates a module's bucket if it is absent and brings its options to match if it is
+// present (novox/hq ADR 0201).
+//
+// **An update, never a delete and recreate**, for the reason a stream is updated: recreating
+// discards what the bucket holds, and what a module's state holds is data. The mesh's caps are
+// asserted with the owner's options, so a bucket made by hand converges to them.
+func (j *JetStream) EnsureBucket(b Bucket) error {
+	history := b.History
+	if history == 0 {
+		history = 1
+	}
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket:       b.Bucket(),
+		Description:  b.Why(),
+		History:      uint8(history),
+		TTL:          time.Duration(b.TTLSeconds) * time.Second,
+		MaxValueSize: StateMaxValueBytes,
+		MaxBytes:     StateMaxBytes,
+		Storage:      jetstream.FileStorage,
+	}); err != nil {
+		return fmt.Errorf("asserting bucket %s: %w", b.Bucket(), err)
+	}
+	return nil
+}
+
+// BucketNames is every key-value bucket on the server, the mesh's and anybody else's.
+func (j *JetStream) BucketNames() ([]string, error) {
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	lister := js.KeyValueStoreNames(ctx)
+	var out []string
+	for name := range lister.Name() {
+		out = append(out, name)
+	}
+	return out, lister.Error()
+}

internal/broker/membership.go

@@ -45,6 +45,11 @@ type Membership struct {
 	// module that must tell the mesh from the world, the route proxy serving an internal name, reads
 	// it here rather than keeping a definition of its own.
 	Mesh []string `json:"mesh,omitempty"`
+	// State is every bucket this module's code may reach, by the name it uses for each, and whether
+	// it may write it (novox/hq ADR 0201): the runtime answers a bundle's state verbs from this list
+	// and refuses, with the reason, what is not on it — the bus enforces only the union over every
+	// module on the machine.
+	State []StateIssued `json:"state,omitempty"`
 }
 
 // Served is one address a tool is answered on.
@@ -103,6 +108,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership {
 			m.Seats = append(m.Seats, SeatServed{Seat: s.Name, Verb: verb, Subject: seatToolSubject(s, verb, node)})
 		}
 	}
+	m.State = stateIssuedFor(d)
 	if len(d.Invokes) > 0 {
 		m.Reaches = map[string][]string{}
 		for _, t := range d.Invokes {

internal/broker/nats.go

@@ -103,6 +103,12 @@ type Principal struct {
 	// permission and nothing beside it.
 	Invokes []string
 
+	// State is the local names of the state this principal's module keeps, and Reads the state of
+	// others it reads as `<module>.<name>` (novox/hq ADR 0201): a bucket each, kept by the owner's
+	// instances and read by whoever declares it.
+	State []string
+	Reads []string
+
 	// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
 	// and never appears here: this file is written to a node's disk and read by a server, and a
 	// secret that can be read from a configuration file is a secret with a wider blast radius
@@ -118,6 +124,10 @@ type Principal struct {
 // goes with the retired seat row.
 var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
 
+// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
+// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
+var perMachineEvents = map[string]bool{"paused.*": true}
+
 // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
 // controller may answer an enrolment is derived from the same constant the user is named from.
 const enrolmentPrefix = "enrol"
@@ -215,6 +225,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// the role, and whichever machine holding it is idle takes it.
 		for _, seat := range seatsTheControllerAsks {
 			pub = append(pub, "mesh.seat."+seat+".accept.>")
+			// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
+			// building, kill it, pause it, resume it. The queue is the controller's to show and to
+			// change, and what one machine is doing with an ask it took only that machine can say.
+			pub = append(pub, "mesh.seat."+seat+".tool.>")
 		}
 		// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
 		// facts under the seat it holds, because a role's events belong to the role and keep their
@@ -239,10 +253,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
 		sub = append(sub, announcing(ControllerSeat)...)
 
-		// The two events it reacts to, and its ack subject on the stream they arrive from
+		// The events it reacts to, and its ack subject on the stream they arrive from
 		// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
 		// controller a subscriber to every event in the mesh, and its permission list would stop
-		// saying what it is for. The ack grant below is scoped per stream because the controller's
+		// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
+		// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
 		// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
 		// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
 		// ever, refused by the list it already has.
@@ -398,10 +413,26 @@ func PermissionsFor(p Principal) (Permissions, error) {
 				"$JS.API.CONSUMER.INFO."+stream+"."+worker,
 				"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
 				"$JS.ACK."+stream+"."+worker+".>")
+			// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
+			// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
+			// moment the controller cancelled it ends it instead of building it. Read, never written:
+			// the set is the controller's, and only the work queues the controller asks — and so may
+			// cancel from — have one.
+			if hasCancelledSet(s.Name) {
+				set := CancelledSetName(s.Name)
+				pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>")
+			}
 			for _, a := range s.Accepts {
 				sub = append(sub, seatSubject(s, "accept", a))
 			}
 			for _, e := range s.Emits {
+				// **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped
+				// seat, an event about the holder itself carries the machine as its last token, and
+				// each holder is granted its own machine's alone.
+				if s.Scope == "node" && p.Node != "" && perMachineEvents[e] {
+					pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node))
+					continue
+				}
 				pub = append(pub, seatSubject(s, "event", e))
 			}
 			for _, t := range s.Serves {
@@ -421,6 +452,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			}
 		}
 
+		// 5. Its state, and the state of others it reads (novox/hq ADR 0201): every one read and
+		//    watched, its own written too.
+		pub = append(pub, stateGrants(p.Module, p.State, p.Reads)...)
+
 	case KindNodeTools:
 		// **One process serves what every module on the machine would have served for itself**
 		// (novox/hq ADR 0175). Each carried module's whole tool namespace — the same grant that
@@ -487,6 +522,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
 				"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+durable,
 				"$JS.ACK."+stream+"."+durable+".>")
 		}
+		// **And it keeps and reads state for the modules it carries** (novox/hq ADR 0201): the union
+		// of what each may do with a bucket — an owner's write, a reader's read. That one module's code
+		// does not write another's bucket through it is the runtime's to keep, from the membership
+		// each assignment is issued, as it keeps each module's events under that module's own name.
+		for _, d := range p.Carries {
+			pub = append(pub, stateGrants(d.Module, stateNames(d.State), d.Reads)...)
+		}
 		sub = unique(sub)
 		pub = unique(pub)
 	}

internal/broker/seattools_test.go

@@ -5,16 +5,16 @@ import "testing"
 // A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
 // node-scoped seat derive two addresses, and a user of the seat may publish any node's.
 func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
-	seat := Seat{Name: "node-dns-resolver", Scope: "node", Serves: []string{"lookup"}}
+	seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
-	one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
+	one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
-	two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
+	two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
-	has(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.one")
+	has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
-	has(t, two.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
+	has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
-	hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup")
+	hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
-	hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
+	hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
 
 	user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
-	has(t, user.Publish, "mesh.seat.node-dns-resolver.tool.lookup.*")
+	has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
 }
 
 // A mesh-scoped seat's tool stays flat: nothing about it changes.

internal/broker/state.go

@@ -0,0 +1,170 @@
+package broker
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// A module's state on the bus (novox/hq ADR 0201, design 32 §4, design 25 §3).
+//
+// A module names the state it keeps (`state`) and the state of others it reads (`reads`), and each
+// is a key-value bucket: the server's own last-per-subject stream with direct reads, delete markers
+// and watches, which is the state relationship the mesh already uses for declarations, opened to
+// modules. The controller creates every bucket from the catalogue — from registration, like a
+// seat's stream, so a reader may watch before the owner runs anywhere — and no module can.
+//
+// Pure, like everything else in this package that decides what the bus holds; jetstream.go is the
+// part that asks a server.
+
+// The mesh's caps on a bucket, the same for every module: a value is a piece of state, not a file,
+// and a bucket that grew without bound would be one module filling the bus's disk for everyone.
+const (
+	StateMaxValueBytes = 256 * 1024
+	StateMaxBytes      = 64 * 1024 * 1024
+)
+
+// A Bucket is one module's declared state as the bus holds it.
+type Bucket struct {
+	Module string
+	Name   string
+	// History is how many values a key keeps; zero is one.
+	History int
+	// TTLSeconds is how long a value lives; zero is until replaced or deleted.
+	TTLSeconds int
+}
+
+// BucketName is the bucket a module's state lives in: the module and the local name joined by an
+// underscore, which neither may contain, so two modules can never derive one bucket.
+func BucketName(module, name string) string { return module + "_" + name }
+
+// Bucket is this bucket's name on the bus.
+func (b Bucket) Bucket() string { return BucketName(b.Module, b.Name) }
+
+// Why is carried into the server's description of the bucket, so somebody reading the server's
+// own state finds whose it is and why it is kept.
+func (b Bucket) Why() string {
+	return fmt.Sprintf("%s's state %q (novox/hq ADR 0201): its current value per key, written by %s, "+
+		"read by whatever declares it reads it; kept when %s is unassigned, because it is data",
+		b.Module, b.Name, b.Module, b.Module)
+}
+
+// bucketOfRead is the bucket a read names, `<module>.<name>`, or false when it names none.
+func bucketOfRead(read string) (string, bool) {
+	at := strings.LastIndex(read, ".")
+	if at <= 0 || at == len(read)-1 {
+		return "", false
+	}
+	module, name := read[:at], read[at+1:]
+	if !safeSubject.MatchString(module) || !safeSubject.MatchString(name) {
+		return "", false
+	}
+	return BucketName(module, name), true
+}
+
+// stateGrants is what a principal publishes to reach the state its modules keep and read: for every
+// bucket, binding to it, reading a key directly, and an ordered consumer for listing and watching,
+// created and deleted on the bucket's own stream, with its flow control answered; for a bucket an
+// owner keeps, writing under the bucket's own subjects too.
+//
+// **Measured against a running server, 2026-10-04** (novox/hq research 024), and each one is there
+// because leaving it out failed: without STREAM.INFO nothing binds; without DIRECT.GET nothing is
+// read; without CONSUMER.CREATE no key is listed and nothing is watched; without CONSUMER.DELETE a
+// watch cannot be stopped and lingers on the server. A write outside these is refused by the server
+// — and reaches the writer as a timeout, not a refusal, which is why the runtime refuses first.
+func stateGrants(module string, keeps []string, reads []string) []string {
+	var out []string
+	read := func(bucket string) {
+		stream := "KV_" + bucket
+		out = append(out,
+			"$JS.API.STREAM.INFO."+stream,
+			"$JS.API.DIRECT.GET."+stream+".>",
+			"$JS.API.CONSUMER.CREATE."+stream+".>",
+			"$JS.API.CONSUMER.DELETE."+stream+".>",
+			"$JS.FC."+stream+".>")
+	}
+	for _, name := range keeps {
+		if !safeSubject.MatchString(name) {
+			continue
+		}
+		bucket := BucketName(module, name)
+		read(bucket)
+		out = append(out, "$KV."+bucket+".>")
+	}
+	for _, r := range reads {
+		if bucket, ok := bucketOfRead(r); ok {
+			read(bucket)
+		}
+	}
+	return out
+}
+
+// StateIssued is one bucket an assignment may reach, by the name its module uses for it: its own
+// state by the local name, another's as `<module>.<name>` (novox/hq ADR 0201).
+type StateIssued struct {
+	Name   string `json:"name"`
+	Bucket string `json:"bucket"`
+	Writes bool   `json:"writes,omitempty"`
+}
+
+// stateIssuedFor is every bucket a module's code may reach, as its membership lists them.
+func stateIssuedFor(d Declared) []StateIssued {
+	var out []StateIssued
+	for _, b := range d.State {
+		out = append(out, StateIssued{Name: b.Name, Bucket: BucketName(d.Module, b.Name), Writes: true})
+	}
+	for _, r := range d.Reads {
+		if bucket, ok := bucketOfRead(r); ok {
+			out = append(out, StateIssued{Name: r, Bucket: bucket})
+		}
+	}
+	return out
+}
+
+// stateNames is the local names of a module's own buckets.
+func stateNames(buckets []Bucket) []string {
+	out := make([]string, 0, len(buckets))
+	for _, b := range buckets {
+		out = append(out, b.Name)
+	}
+	return out
+}
+
+// A BucketAsserter is the part of a JetStream connection bucket assertion needs.
+type BucketAsserter interface {
+	// EnsureBucket creates the bucket if absent and brings its options to match if present, never
+	// discarding what it holds.
+	EnsureBucket(b Bucket) error
+	// BucketNames is every key-value bucket on the server.
+	BucketNames() ([]string, error)
+}
+
+// RaiseBuckets asserts every declared bucket and answers the buckets on the server that nothing
+// declares any more.
+//
+// **Those are reported, never removed** (novox/hq ADR 0201, ADR 0030): what a module stored is
+// data, and a manifest edited, a module renamed or a catalogue entry dropped is an ordinary day's
+// work that must not take data with it. Removing one is a person's act.
+func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err error) {
+	sorted := append([]Bucket(nil), buckets...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Bucket() < sorted[j].Bucket() })
+	declared := map[string]bool{}
+	for _, b := range sorted {
+		if err := a.EnsureBucket(b); err != nil {
+			return nil, fmt.Errorf("asserting %s's state %q: %w", b.Module, b.Name, err)
+		}
+		declared[b.Bucket()] = true
+	}
+	names, err := a.BucketNames()
+	if err != nil {
+		return nil, fmt.Errorf("listing the bus's state: %w", err)
+	}
+	for _, n := range names {
+		// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
+		if !declared[n] && !IsCancelledSet(n) {
+			undeclared = append(undeclared, n)
+		}
+	}
+	sort.Strings(undeclared)
+	return undeclared, nil
+}

internal/broker/state_test.go

@@ -0,0 +1,180 @@
+package broker
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+)
+
+// The grants measured against a running server (novox/hq research 024): an owner reads and writes
+// its bucket, a reader only reads, and neither reaches any other bucket.
+func TestAnOwnerWritesItsStateAndAReaderOnlyReads(t *testing.T) {
+	owner, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "claude-code",
+		State: []string{"servers"}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range []string{
+		"$KV.claude-code_servers.>",
+		"$JS.API.STREAM.INFO.KV_claude-code_servers",
+		"$JS.API.DIRECT.GET.KV_claude-code_servers.>",
+		"$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>",
+		"$JS.API.CONSUMER.DELETE.KV_claude-code_servers.>",
+		"$JS.FC.KV_claude-code_servers.>",
+	} {
+		has(t, owner.Publish, s)
+	}
+	hasNot(t, owner.Publish, "$KV.>")
+	hasNot(t, owner.Publish, "$JS.API.>")
+
+	reader, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "console",
+		Reads: []string{"claude-code.servers"}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	has(t, reader.Publish, "$JS.API.DIRECT.GET.KV_claude-code_servers.>")
+	has(t, reader.Publish, "$JS.API.CONSUMER.CREATE.KV_claude-code_servers.>")
+	hasNot(t, reader.Publish, "$KV.claude-code_servers.>")
+	for _, s := range reader.Subscribe {
+		if s == "$KV.claude-code_servers.>" {
+			t.Fatalf("a reader subscribes the bucket's subjects directly: %v", reader.Subscribe)
+		}
+	}
+}
+
+// One runtime carries every module on its machine, so its grant is the union: the owner's write
+// where an owner is carried, a read where only a reader is.
+func TestTheRuntimeKeepsAndReadsStateForItsModules(t *testing.T) {
+	perms, err := PermissionsFor(Principal{Kind: KindNodeTools, Node: "one", Module: RuntimeModule,
+		Carries: []Declared{
+			{Module: "claude-code", State: []Bucket{{Module: "claude-code", Name: "servers"}},
+				Reads: []string{"licence-manager.bindings"}},
+			{Module: "audit"},
+		}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	has(t, perms.Publish, "$KV.claude-code_servers.>")
+	has(t, perms.Publish, "$JS.API.DIRECT.GET.KV_licence-manager_bindings.>")
+	hasNot(t, perms.Publish, "$KV.licence-manager_bindings.>")
+}
+
+// A module with no state is granted nothing of any bucket — the composition of every module that
+// existed before this is unchanged.
+func TestAModuleWithNoStateReachesNoBucket(t *testing.T) {
+	perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing",
+		Emits: []string{"order.placed"}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range perms.Publish {
+		if strings.HasPrefix(s, "$KV.") || strings.HasPrefix(s, "$JS.FC.") || strings.Contains(s, ".KV_") {
+			t.Fatalf("granted %q without declaring state", s)
+		}
+	}
+}
+
+// A read that names no bucket grants nothing rather than something that happens to parse.
+func TestAReadThatNamesNoBucketGrantsNothing(t *testing.T) {
+	if got := stateGrants("a", nil, []string{"nodot", "x.", ".y", "a.b>"}); len(got) != 0 {
+		t.Fatalf("granted %v for reads that name no bucket", got)
+	}
+}
+
+// The membership lists every bucket the module's code may reach, by the name the module uses for
+// it, and whether it may write it — the list the runtime refuses from.
+func TestAMembershipListsTheStateItsModuleMayReach(t *testing.T) {
+	m := MembershipFor("one", Declared{Module: "claude-code",
+		State: []Bucket{{Module: "claude-code", Name: "servers"}},
+		Reads: []string{"licence-manager.bindings"}}, Placements{})
+	want := []StateIssued{
+		{Name: "servers", Bucket: "claude-code_servers", Writes: true},
+		{Name: "licence-manager.bindings", Bucket: "licence-manager_bindings"},
+	}
+	if !slices.Equal(m.State, want) {
+		t.Fatalf("issued %+v, want %+v", m.State, want)
+	}
+	if none := MembershipFor("one", Declared{Module: "audit"}, Placements{}); none.State != nil {
+		t.Fatalf("a module with no state was issued %+v", none.State)
+	}
+}
+
+type buckets struct {
+	ensured []string
+	on      []string
+}
+
+func (b *buckets) EnsureBucket(x Bucket) error {
+	b.ensured = append(b.ensured, x.Bucket())
+	return nil
+}
+func (b *buckets) BucketNames() ([]string, error) { return b.on, nil }
+
+// Every declared bucket is asserted; one on the server that nothing declares is said, not removed.
+func TestRaisingStateReportsWhatNothingDeclares(t *testing.T) {
+	b := &buckets{on: []string{"claude-code_servers", "gone_old", "ours_by_hand"}}
+	undeclared, err := RaiseBuckets(b, []Bucket{{Module: "claude-code", Name: "servers"}, {Module: "a", Name: "b"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(b.ensured, []string{"a_b", "claude-code_servers"}) {
+		t.Fatalf("asserted %v", b.ensured)
+	}
+	if !slices.Equal(undeclared, []string{"gone_old", "ours_by_hand"}) {
+		t.Fatalf("reported %v", undeclared)
+	}
+}
+
+// Against a real server: a bucket is created with the owner's options and the mesh's caps,
+// asserting it again changes nothing and keeps what it holds, and a changed option is brought to
+// match in place.
+func TestABucketIsAssertedInPlace(t *testing.T) {
+	js := aLiveBus(t)
+	b := Bucket{Module: "statetest", Name: "servers"}
+	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
+		t.Fatalf("a real server refused a module's bucket: %v", err)
+	}
+	kv, err := js.Context().KeyValue(b.Bucket())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil {
+		t.Fatal(err)
+	}
+	b.History = 3
+	if _, err := RaiseBuckets(js, []Bucket{b}); err != nil {
+		t.Fatalf("asserting the bucket again failed, so a restart would: %v", err)
+	}
+	got, err := kv.Get("all.one")
+	if err != nil || string(got.Value()) != `{"kept":true}` {
+		t.Fatalf("asserting again lost what the bucket held: %v %v", got, err)
+	}
+	status, err := kv.Status()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status.History() != 3 {
+		t.Fatalf("history is %d, the owner declared 3", status.History())
+	}
+	if s, ok := status.(*nats.KeyValueBucketStatus); ok {
+		if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes {
+			t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes)
+		}
+	}
+}
+
+// Against a real server: the handle over a connection the control plane already holds asserts a
+// bucket as Dial's does — what a push uses, so a module registered since the last start has its
+// bucket before its membership names it.
+func TestABucketIsAssertedOverAHeldConnection(t *testing.T) {
+	js := aLiveBus(t)
+	held := OnConn(js.Conn())
+	if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil {
+		t.Fatalf("asserting over a held connection failed: %v", err)
+	}
+	if _, err := js.Context().KeyValue("statetest_held"); err != nil {
+		t.Fatalf("the bucket is not there: %v", err)
+	}
+}

internal/broker/streams.go

@@ -226,8 +226,23 @@ var ControllerFollows = []string{
 	// registers build-agent itself comes from there. Appended, for the same reason as above; goes
 	// with the retired seat row.
 	seatEventSubject("mesh-build-machine", "built"),
+	// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
+	// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
+	// from whichever module provides — because the rule is about every provider, and a list of
+	// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
+	// failed every consumer for a day and only its journal said so (issue 179). Appended, because
+	// the index is a name.
+	moduleEventSubject("*", ProvisionerFailing),
+	moduleEventSubject("*", ProvisionerRecovered),
 }
 
+// The provider standing events, by their local names. Written here as well as in the catalogue
+// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
+const (
+	ProvisionerFailing   = "provisioner.failing"
+	ProvisionerRecovered = "provisioner.recovered"
+)
+
 // moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
 // what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
 func moduleEventSubject(module, event string) string {
@@ -270,7 +285,8 @@ func MeshConsumers() []Consumer {
 			// announcement handed over behind it must wait on the server, not time out on the
 			// client and come back to be acted on again.
 			MaxAckPending: 1,
-			Why: "the two events the mesh's own controller reacts to, one at a time; after " +
+			FromNow:       true,
+			Why: "the events the mesh's own controller reacts to, one at a time; after " +
 				"max-deliver it dead-letters, because an announcement it cannot act on will not " +
 				"become actionable",
 		},

internal/broker/testdata/composed.conf

@@ -24,8 +24,8 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
-        subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
+        subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
       { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {

internal/broker/users.go

@@ -33,6 +33,14 @@ type Declared struct {
 	Watches []Seat
 	// Invokes are the tools it calls, `<module>.<tool>` or `*` (novox/hq ADR 0152).
 	Invokes []string
+	// State is the state it keeps, each a bucket its instances write (novox/hq ADR 0201).
+	State []Bucket
+	// Reads are other modules' state it reads, each `<module>.<name>` (novox/hq ADR 0201).
+	Reads []string
+	// NoAccount says the module declares no own secret named broker, so no account could ever be
+	// delivered to it and nothing can connect as it (novox/hq issue 195). Said in the negative so a
+	// record that does not say is composed as it always was.
+	NoAccount bool
 }
 
 // Records is what composing a user list needs to know about the mesh, and nothing more.
@@ -77,10 +85,20 @@ func Users(r Records) ([]Principal, error) {
 			if runtimeHere && d.Module == RuntimeModule {
 				continue
 			}
+			// **A module with nowhere to read an account is no user** (novox/hq issue 195). `module
+			// issue` refuses it one (issue 078: an account nothing reads is an orphan), so its user
+			// could only ever be left out of the file for want of a password — and every such module
+			// was named, on every status and plan, as a credential the mesh had not minted. Where the
+			// runtime is, it speaks for the module; where it is not, the module cannot speak at all,
+			// and a user would not change that.
+			if d.NoAccount {
+				continue
+			}
 			out = append(out, Principal{
 				Kind: KindModule, Node: node, Module: d.Module,
 				Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
 				Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
+				State: stateNames(d.State), Reads: d.Reads,
 			})
 		}
 		if runtimeHere {

internal/broker/users_test.go

@@ -296,3 +296,57 @@ func TestTheRuntimeModuleBecomesTheMachinesRuntimePrincipal(t *testing.T) {
 		t.Error("telegram lost its own principal when the runtime arrived on its node")
 	}
 }
+
+// A module that declares nowhere to read an account is no user: it could never be issued one, so it
+// was only ever named as a credential the mesh had not minted (novox/hq issue 195). Where the runtime
+// is, the runtime still carries it — its grants are the runtime's.
+func TestAModuleThatCannotReadAnAccountIsNoUser(t *testing.T) {
+	r := someRecords()
+	r.Assigned["one"] = append(r.Assigned["one"],
+		Declared{Module: "packet-filter", NoAccount: true, Emits: []string{"rule.changed"}},
+		Declared{Module: RuntimeModule})
+	users, err := Users(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, u := range users {
+		if u.Username() == "one.packet-filter" {
+			t.Fatalf("packet-filter declares no broker secret and was composed as a user: %v", namesOf(t, r))
+		}
+		if u.Kind == KindNodeTools {
+			carried := false
+			for _, d := range u.Carries {
+				carried = carried || d.Module == "packet-filter"
+			}
+			if !carried {
+				t.Error("the runtime stopped carrying a module that has no account of its own")
+			}
+		}
+	}
+	if names := strings.Join(namesOf(t, r), ","); !strings.Contains(names, "one.telegram") {
+		t.Errorf("a module that does read an account lost its user: %s", names)
+	}
+}
+
+// A carried module with no account still has its consumer made: the runtime reads it on the module's
+// behalf (ADR 0198), and the consumer was derived from the module's own user until issue 195 took
+// that user away.
+func TestACarriedModuleWithNoAccountStillHasItsConsumer(t *testing.T) {
+	r := someRecords()
+	r.Assigned["one"] = append(r.Assigned["one"],
+		Declared{Module: "listener", NoAccount: true, Consumes: []string{"shop.order.placed"}},
+		Declared{Module: RuntimeModule})
+	r.Assigned["two"] = append(r.Assigned["two"],
+		Declared{Module: "auditor", Consumes: []string{"shop.order.placed"}})
+	users, err := Users(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := map[string]int{}
+	for _, c := range ConsumersOf(users) {
+		got[c.Node+"/"+c.Module]++
+	}
+	if got["one/listener"] != 1 || got["two/auditor"] != 1 || len(got) != 2 {
+		t.Fatalf("consumers: %v", got)
+	}
+}

internal/builder/builder.go

@@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
 		sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
 		for _, a := range artifacts {
 			say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
-			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
+			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
 			if err != nil {
 				say("artifact", "%s FAILED: %v", a.Name, err)
 				return Result{}, err
@@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
 
 func one(ctx context.Context, run Runner, publish Publisher,
 	module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
-	held map[string]string, npmrc string, seats map[string]string,
+	held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
 	say func(step, format string, args ...any)) (catalogue.Built, error) {
 
 	switch a.Kind {
@@ -582,6 +582,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
 					"holds no copy of it. Build %s first",
 				module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
 		}
+		// The module's own packages first, where the compiler and the bundler resolve them from
+		// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
+		if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
+			return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+		}
 		say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
 		compiled, err := compile(ctx, run, tree, chain, base, a)
 		if err != nil {
@@ -756,6 +761,9 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err
 	tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
 	cmd := exec.CommandContext(ctx, name, args...)
 	cmd.Dir = dir
+	// **Ended whole when the build is** (novox/hq ADR 0219): its own process group, killed as one
+	// when the context ends, so a build killed by hand leaves no `git` or `docker` child running.
+	inItsOwnGroup(cmd)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		tell("run", "! %s %s failed after %s", name, args[0], since(started))

internal/builder/dependencies.go

@@ -0,0 +1,147 @@
+package builder
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// A module's own packages, installed before its bundle is compiled, so the bundler inlines them.
+//
+// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler
+// resolve an import by walking up from the module's source: the module's own directory first, then
+// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a
+// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose
+// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle").
+// Now the module's `package.json` says what it depends on, as any Node package does, and the build
+// installs exactly that into the module's own directory before compiling.
+//
+// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says**
+// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A
+// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a
+// workstation — and installing that range would shadow the toolchain's copy for this module alone:
+// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads.
+// So the SDK is taken out of what is installed (and never fetched), and any copy something else
+// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the
+// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin.
+//
+// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile
+// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept
+// between builds. What is shared is npm's own download cache, a named volume, which is
+// content-addressed and verified by integrity on every read — it saves the network, never the
+// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script
+// would build natively could not be inlined into one file anyway.
+
+// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a
+// module's own dependencies never supply (above).
+const sdkPackage = "@novox/mesh-sdk"
+
+// npmCache is the named volume npm's download cache lives in across builds on one build node.
+const npmCache = "mesh-builder-npm-cache"
+
+// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when
+// the module has no package.json or depends on nothing else — which builds exactly as before.
+func ownDependencies(tree string) ([]string, error) {
+	raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var p struct {
+		Dependencies map[string]string `json:"dependencies"`
+	}
+	if err := json.Unmarshal(raw, &p); err != nil {
+		return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
+	}
+	var names []string
+	for name := range p.Dependencies {
+		if name != sdkPackage {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// installSteps is the script run inside the toolchain image, from the module's own directory ($0).
+// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes
+// the SDK out of what is installed, installs production dependencies only, removes any copy of the
+// SDK something pulled in, and puts the result at the module's node_modules.
+const installSteps = `set -e
+work="$(mktemp -d)"
+cp "$0/package.json" "$work/"
+if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi
+cd "$work"
+node -e '
+const fs = require("fs"), sdk = process.argv[1];
+const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
+for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk];
+delete p.devDependencies; delete p.scripts;
+fs.writeFileSync("package.json", JSON.stringify(p));
+' "$1"
+shift
+if [ -f package-lock.json ]; then
+  npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@"
+else
+  npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@"
+fi
+find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} +
+rm -rf "$0/node_modules"
+cp -a node_modules "$0/node_modules"
+`
+
+// installOwn installs a TypeScript module's own production dependencies into its directory, in the
+// toolchain image, before the compile — or does nothing at all for a module that has none.
+func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
+	registry Npmrc, say func(step, format string, args ...any)) error {
+	if chain.Language != "typescript" {
+		return nil
+	}
+	deps, err := ownDependencies(tree)
+	if err != nil || len(deps) == 0 {
+		return err
+	}
+	scoped := strings.TrimSpace(registry.Scope)
+	if !registry.Enabled() {
+		// **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on
+		// the public one, where anybody may have published it: a dependency that installs is not
+		// the dependency the module meant.
+		for _, d := range deps {
+			if strings.HasPrefix(d, "@novox/") {
+				return fmt.Errorf("the module depends on %s, and this build knows no package registry "+
+					"for its scope; it would resolve from the public registry, which is not where the "+
+					"mesh publishes it", d)
+			}
+		}
+	}
+	const within = "/app/modules/module"
+	invocation := []string{"run", "--rm",
+		"--volume", tree + ":" + within,
+		"--volume", npmCache + ":/root/.npm",
+		"--workdir", within}
+	var flags []string
+	if registry.Enabled() {
+		// The registry is reached where the binding says it is, which may be this machine's own
+		// loopback — the reason an image build that resolves packages runs on the host network too.
+		invocation = append(invocation, "--network", "host")
+		reg := strings.TrimSpace(registry.Registry)
+		if !strings.HasSuffix(reg, "/") {
+			reg += "/"
+		}
+		flags = append(flags, "--"+scoped+":registry="+reg)
+	}
+	invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage)
+	invocation = append(invocation, flags...)
+	say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", "))
+	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
+		return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err)
+	}
+	return nil
+}

internal/builder/dependencies_test.go

@@ -0,0 +1,131 @@
+package builder
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// A module's own packages (dependencies.go): installed into its own directory, in the toolchain,
+// before the compile, so the bundler inlines them — the SDK always the toolchain's.
+
+func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) {
+	t.Helper()
+	files := map[string]string{"index.ts": "console.log(1)"}
+	if pkg != "" {
+		files["package.json"] = pkg
+	}
+	for k, v := range extra {
+		files[k] = v
+	}
+	r, workspace := aRepository(t, aBundle, files)
+	held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
+	_, err := Build(context.Background(), compiling{r}.run, r,
+		"https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil)
+	return r, err
+}
+
+func installs(r *recorded) []string {
+	var out []string
+	for _, line := range r.ran {
+		if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") {
+			out = append(out, line)
+		}
+	}
+	return out
+}
+
+func compileIndex(r *recorded) int {
+	for i, line := range r.ran {
+		if strings.Contains(line, "--outDir") {
+			return i
+		}
+	}
+	return -1
+}
+
+func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`,
+		nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := installs(r)
+	if len(got) != 1 {
+		t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n"))
+	}
+	line := got[0]
+	for _, want := range []string{
+		"mesh-tools/build@sha256:",       // in the toolchain image
+		":/app/modules/module",           // into the module's own directory
+		"--workdir /app/modules/module",  //
+		npmCache + ":/root/.npm",         // npm's verified download cache, and only that
+		"--omit=dev", "--ignore-scripts", // production packages, no build-node scripts
+		"npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges
+		"--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry
+		"--network host",
+		"@novox/mesh-sdk", // named, to be taken out of what is installed
+	} {
+		if !strings.Contains(line, want) {
+			t.Errorf("the install lacks %q:\n%s", want, line)
+		}
+	}
+	// The SDK is the toolchain's: never installed from the module's range, and any copy removed.
+	if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) {
+		t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line)
+	}
+	if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 ||
+		i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n"))
+	}
+}
+
+// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install.
+func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) {
+	without, err := buildWithPackageJSON(t, "", nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range []string{
+		`{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`,
+		`{"type":"module"}`,
+	} {
+		with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(strings.Join(with.ran, "\n"), "npm ") {
+			t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n"))
+		}
+		if len(with.ran) != len(without.ran) {
+			t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s",
+				strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n"))
+		}
+	}
+}
+
+// Without the mesh's registry a scoped package would resolve on the public one: refused by name.
+func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "@novox/other") {
+		t.Fatalf("a scoped package was installed with no registry for its scope: %v", err)
+	}
+	if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatal("the compile ran after the refusal")
+	}
+	// A public package installs without one, from the public registry and nothing else.
+	r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") {
+		t.Fatalf("a public package's install: %v", got)
+	}
+}
+
+func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) {
+	_, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "package.json") {
+		t.Fatalf("a broken package.json was not refused by name: %v", err)
+	}
+}

internal/builder/kill.go

@@ -0,0 +1,78 @@
+package builder
+
+import (
+	"context"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// A build killed by hand (novox/hq ADR 0219).
+//
+// A build is a tree of commands — git, then docker, and docker's own children — and a container the
+// docker client started keeps running when the client dies. So ending one by hand is three things:
+// the build's context is cancelled, which kills each command's whole process group rather than the
+// one process the context knows; every container the build started carries the build's id as a
+// label, so what outlived its client is found and removed by that label; and the holder announces
+// the outcome itself, since nothing else will.
+
+// BuildLabel is the label every container a build starts carries, valued with the build's id.
+const BuildLabel = "mesh.build"
+
+// KillWait is how long a command killed with its build may take to let go of its output before it
+// is abandoned: a grandchild holding the pipe open must not hold the build open with it.
+const KillWait = 10 * time.Second
+
+// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its
+// context ends.
+func inItsOwnGroup(cmd *exec.Cmd) {
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	cmd.Cancel = func() error {
+		if cmd.Process == nil {
+			return nil
+		}
+		// The negative pid is the group: the command and everything it started.
+		if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
+			return cmd.Process.Kill()
+		}
+		return nil
+	}
+	cmd.WaitDelay = KillWait
+}
+
+// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
+// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
+// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
+func Labelled(run Runner, id string) Runner {
+	return func(ctx context.Context, dir string, name string, args ...string) (string, error) {
+		return run(ctx, dir, name, LabelledArgs(name, args, id)...)
+	}
+}
+
+// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`.
+func LabelledArgs(name string, args []string, id string) []string {
+	if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
+		return args
+	}
+	out := make([]string, 0, len(args)+2)
+	out = append(out, "run", "--label", BuildLabel+"="+id)
+	return append(out, args[1:]...)
+}
+
+// RemoveContainersOf removes every container labelled with the build's id, running or not, and
+// says how many. Run with a context of its own: the build's is the one that was just cancelled.
+func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) {
+	out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id)
+	if err != nil {
+		return 0, err
+	}
+	ids := strings.Fields(out)
+	if len(ids) == 0 {
+		return 0, nil
+	}
+	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+		return 0, err
+	}
+	return len(ids), nil
+}

internal/builder/kill_test.go

@@ -0,0 +1,103 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills
+// the command's whole process group, not the one process the context knows about.
+func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) {
+	dir := t.TempDir()
+	child := filepath.Join(dir, "child")
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	began := time.Now()
+	go func() {
+		// A shell that starts a grandchild and waits on it: killing the shell alone would leave the
+		// grandchild running, holding the output open.
+		_, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`)
+		done <- err
+	}()
+	var pid int
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" {
+			pid, _ = strconv.Atoi(strings.TrimSpace(string(raw)))
+			break
+		}
+	}
+	if pid == 0 {
+		t.Fatal("the command never started its child")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("a killed command reported success")
+		}
+	case <-time.After(KillWait + 5*time.Second):
+		t.Fatal("the killed command never returned")
+	}
+	if took := time.Since(began); took > KillWait {
+		t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took)
+	}
+	for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) {
+			return
+		}
+	}
+	_ = syscall.Kill(pid, syscall.SIGKILL)
+	t.Fatalf("the grandchild %d outlived the kill", pid)
+}
+
+// Every `docker run` a build starts carries its id as a label; nothing else is touched.
+func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) {
+	got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
+	if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("docker run became %v", got)
+	}
+	for _, c := range []struct {
+		name string
+		args []string
+	}{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} {
+		if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) {
+			t.Errorf("%s %v became %v", c.name, c.args, got)
+		}
+	}
+	var ran [][]string
+	run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, append([]string{name}, args...))
+		return "", nil
+	}, "build-2")
+	_, _ = run(context.Background(), "", "docker", "run", "img")
+	if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}
+
+// What a kill removes is found by the label, and only what it finds.
+func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
+	var ran []string
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\nc2\n", nil
+		}
+		return "", nil
+	}
+	n, err := RemoveContainersOf(context.Background(), run, "build-3")
+	if err != nil || n != 2 {
+		t.Fatalf("%d %v", n, err)
+	}
+	if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}

internal/builder/registry.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"net/http"
 	"strings"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
 )
 
 // Where built artifacts go.
@@ -66,22 +68,32 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
 	return pinned, nil
 }
 
-// PublishArchive stores bytes as a blob and returns where to fetch them from.
+// PublishArchive stores bytes as a blob, holds it by a manifest, and returns where to fetch them
+// from.
 //
-// Two steps, which is the registry's own protocol: ask for somewhere to put it, then put it there
+// Two steps for the blob, which is the registry's own protocol: ask for somewhere to put it, then
-// naming the digest. The registry verifies the digest itself, so a blob that arrived corrupted is
+// put it there naming the digest. The registry verifies the digest itself, so a blob that arrived
-// refused by the thing storing it rather than by the machine unpacking it a week later.
+// corrupted is refused by the thing storing it rather than by the machine unpacking it a week
+// later.
+//
+// **Then a manifest that names it** (novox/hq issue 253, ADR 0189). The store's own collector
+// marks only from manifests, and a blob no manifest names is collected however much the mesh
+// means to keep it — so an archive published bare is an archive the first nightly collection
+// deletes. The holder is composed from the digest and size alone (artifacts.Holder), which is
+// what lets the sweep recompute it to backfill or let go without anything being recorded here.
+// What a machine is told to fetch is the blob, exactly as before.
 func (r Registry) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
 	base := "http://" + r.Address + "/v2/" + repository
 	final := base + "/blobs/" + digest
 
 	// Already there. Blobs are immutable and named by their content, so this is not an
 	// optimisation — re-uploading would be asking the registry to store what it already has under
-	// the name it already has.
+	// the name it already has. It is still held: a blob published before holders existed is
+	// exactly the one a rebuild of the same source finds already there.
 	if there, err := r.has(ctx, final); err != nil {
 		return "", err
 	} else if there {
-		return final, nil
+		return final, r.hold(ctx, repository, digest, len(body))
 	}
 
 	start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
@@ -119,7 +131,17 @@ func (r Registry) PublishArchive(ctx context.Context, repository string, body []
 		said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
 		return "", fmt.Errorf("%s refused the blob: %s %s", base, done.Status, strings.TrimSpace(string(said)))
 	}
-	return final, nil
+	return final, r.hold(ctx, repository, digest, len(body))
+}
+
+// hold puts the manifest holding an archive beside it. A build whose archive could not be held is
+// a failed build: recorded as published, it would be an archive the store's collector takes.
+func (r Registry) hold(ctx context.Context, repository, digest string, size int) error {
+	store := artifacts.Store{Address: r.Address, HTTP: r.client()}
+	if _, err := store.HoldBlob(ctx, repository, digest, int64(size)); err != nil {
+		return fmt.Errorf("published %s/blobs/%s and could not hold it by a manifest: %w", repository, digest, err)
+	}
+	return nil
 }
 
 // has is whether this registry already holds what is at that URL.

internal/builder/registry_test.go

@@ -9,6 +9,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
 )
 
 // An OCI registry as a content-addressed blob store, which is what it is.
@@ -18,9 +20,10 @@ import (
 // there is not sent again, and that a tag is never accepted as a pin.
 
 type fakeRegistry struct {
-	blobs    map[string][]byte
+	blobs     map[string][]byte
-	uploads  int
+	manifests map[string][]byte // "<repository>@<digest>"
-	location string
+	puts      map[string]int    // blob uploads, by digest
+	location  string
 }
 
 func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
@@ -28,6 +31,8 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
 	if f.blobs == nil {
 		f.blobs = map[string][]byte{}
 	}
+	f.manifests = map[string][]byte{}
+	f.puts = map[string]int{}
 	mux := http.NewServeMux()
 	server := httptest.NewServer(mux)
 	mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) {
@@ -39,8 +44,28 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
 				return
 			}
 			w.WriteHeader(http.StatusNotFound)
+		case strings.Contains(r.URL.Path, "/manifests/"):
+			// The archive's holder (novox/hq issue 253): asked for with an Accept, put by digest.
+			repository, digest, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/v2/"), "/manifests/")
+			switch r.Method {
+			case http.MethodHead:
+				if _, ok := f.manifests[repository+"@"+digest]; ok &&
+					strings.Contains(r.Header.Get("Accept"), "application/vnd.oci.image.manifest.v1+json") {
+					w.WriteHeader(http.StatusOK)
+					return
+				}
+				w.WriteHeader(http.StatusNotFound)
+			case http.MethodPut:
+				body, _ := io.ReadAll(r.Body)
+				sum := sha256.Sum256(body)
+				if digest != "sha256:"+hex.EncodeToString(sum[:]) {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				f.manifests[repository+"@"+digest] = body
+				w.WriteHeader(http.StatusCreated)
+			}
 		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/blobs/uploads/"):
-			f.uploads++
 			where := f.location
 			if where == "" {
 				where = "/v2/upload/" + hex.EncodeToString([]byte("session"))
@@ -58,6 +83,7 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
 				return
 			}
 			f.blobs[digest] = body
+			f.puts[digest]++
 			w.WriteHeader(http.StatusCreated)
 		default:
 			w.WriteHeader(http.StatusNotFound)
@@ -92,6 +118,57 @@ func TestAnArchiveIsStoredAndFetchableByItsDigest(t *testing.T) {
 	}
 }
 
+func TestAnArchiveIsPublishedWithAManifestHoldingIt(t *testing.T) {
+	// The store's collector marks only from manifests, so a bare blob is one the first collection
+	// deletes, kept or not (novox/hq issue 253, ADR 0189). Every archive goes out held, by the
+	// holder the sweep can compute for itself from the digest and size.
+	f := &fakeRegistry{}
+	r := registryFor(t, f)
+	body := []byte("a theme")
+	sum := sha256.Sum256(body)
+	digest := "sha256:" + hex.EncodeToString(sum[:])
+
+	where, err := r.PublishArchive(context.Background(), "shell/config", body, digest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasSuffix(where, "/v2/shell/config/blobs/"+digest) {
+		t.Fatalf("a machine is told to fetch %q; the blob is still what is fetched", where)
+	}
+	manifest, holder := artifacts.Holder(digest, int64(len(body)))
+	if got := f.manifests["shell/config@"+holder]; string(got) != string(manifest) {
+		t.Fatalf("the store holds %v; want the holder %s in the archive's own repository", f.manifests, holder)
+	}
+	if !strings.Contains(string(manifest), `"digest":"`+digest+`"`) {
+		t.Fatalf("the holder does not name the archive: %s", manifest)
+	}
+	empty := sha256.Sum256([]byte("{}"))
+	if _, ok := f.blobs["sha256:"+hex.EncodeToString(empty[:])]; !ok {
+		t.Fatal("the holder's empty config was never put, and a registry refuses a manifest without it")
+	}
+}
+
+func TestAnArchiveAlreadyThereIsStillHeld(t *testing.T) {
+	// A rebuild of the same source finds its archive already there — often one published bare,
+	// before holders. It is not sent again, and it is held.
+	f := &fakeRegistry{}
+	r := registryFor(t, f)
+	body := []byte("published bare")
+	sum := sha256.Sum256(body)
+	digest := "sha256:" + hex.EncodeToString(sum[:])
+	f.blobs[digest] = body
+
+	if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
+		t.Fatal(err)
+	}
+	if f.puts[digest] != 0 {
+		t.Fatal("a blob already there was sent again")
+	}
+	if _, holder := artifacts.Holder(digest, int64(len(body))); f.manifests["shell/config@"+holder] == nil {
+		t.Fatal("a blob already there was left bare")
+	}
+}
+
 func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
 	// Not an optimisation: blobs are named by their content, so re-uploading is asking the
 	// registry to store what it already has under the name it already has.
@@ -107,8 +184,11 @@ func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
 	if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
 		t.Fatal(err)
 	}
-	if f.uploads != 1 {
+	if f.puts[digest] != 1 {
-		t.Fatalf("the blob was uploaded %d times", f.uploads)
+		t.Fatalf("the blob was uploaded %d times", f.puts[digest])
+	}
+	if len(f.manifests) != 1 {
+		t.Fatalf("publishing twice left %d holders; want the one", len(f.manifests))
 	}
 }
 

internal/builder/toolchain.go

@@ -67,8 +67,9 @@ type Toolchain struct {
 	// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
 	// bundle with its dependencies and without that line still fails to start — and the pruned,
 	// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
-	// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
+	// dependencies, and nothing module-specific (a module's own npm dependencies are installed into
-	// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
+	// its own directory before the compile and inlined by the bundler: dependencies.go). Empty for a
+	// language whose bundle carries its own —
 	// a Go binary is static, a Python bundle is installed with its dependencies.
 	//
 	// A toolchain image without the directory fails the build by name rather than packing a bundle

internal/catalogue/a_service_process_test.go

@@ -0,0 +1,166 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 213 (ADR 0188 §1, §3): a module's own Go service is a bundle the host runs as a
+// process, not an image. Each test holds one thing that had to change in the composer for the
+// controller to be declared that way.
+
+const aServiceDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
+
+// aServiceModule is the controller's shape in miniature: it answers tools of its own, its code is a
+// Go bundle a process runs as an account it declares, its secrets belong to that account, it
+// prepares its state, and its process replaces the container it used to run as.
+func aServiceModule(t *testing.T) Manifest {
+	t.Helper()
+	raw := `{
+	  "module": "svc", "version": "1", "tools": ["status"], "prepares": true,
+	  "own-secrets": {"store": "${dir:state}/store"},
+	  "secrets-owner": "svc",
+	  "resources": [
+	    {"id": "state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "svc"},
+	    {"id": "service", "type": "process", "name": "svc", "artifact": "code",
+	     "run": ["./svc", "serve"], "user": "svc", "replaces": ["server"],
+	     "env": {"SVC_STORE_FILE": "${dir:state}/store", "SVC_STORE_PORT": "${seat:mesh-store:5432}"}},
+	    {"id": "account", "type": "user", "name": "svc", "shell": "/usr/bin/nologin", "home": "/var/lib/svc"}
+	  ],
+	  "build": {"artifacts": [{"name": "code", "kind": "bundle", "language": "go", "system": "arch",
+	    "from": "cmd/svc", "binary": "svc"}]}
+	}`
+	m, err := ParseManifest([]byte(raw))
+	if err != nil {
+		t.Fatalf("the service's manifest is refused: %v", err)
+	}
+	resolved, err := m.Resolve([]Built{{Name: "code", Kind: ArtifactBundle,
+		Reference: ArtifactStoreScheme + "svc/code@" + aServiceDigest, Digest: aServiceDigest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return resolved
+}
+
+func composeTheService(t *testing.T, with Rendering) []map[string]any {
+	t.Helper()
+	with.Needed = map[string]map[string]string{"svc": {"store": "sealed-store"}}
+	with.ArtifactStore = "anchor.internal:5100"
+	out, err := Resolution{Node: "anchor", Modules: []Manifest{aServiceModule(t)}}.Declaration(with)
+	if err != nil {
+		t.Fatalf("the service does not compose: %v", err)
+	}
+	return out
+}
+
+func indexOf(out []map[string]any, id string) int {
+	for i, r := range out {
+		if r["id"] == id {
+			return i
+		}
+	}
+	return -1
+}
+
+// A module that declares tools has every bundle served by the node's runtime unless it says
+// otherwise — and the controller declares the verbs it answers as tools. Its service bundle is run
+// by its own process; launched a second time by the runtime it would be a second controller
+// pretending to be an MCP server.
+func TestABundleItsOwnProcessRunsIsNotServedByTheRuntime(t *testing.T) {
+	m := aServiceModule(t)
+	if len(m.Bundles) != 1 {
+		t.Fatalf("the service's bundle was not kept: %+v", m.Bundles)
+	}
+	if loads := m.Bundles[0].Loads; len(loads) != 0 {
+		t.Fatalf("the runtime would launch the service's own bundle as tools: %v", loads)
+	}
+	// And a bundle no resource runs still is served, as a module declaring tools always had it.
+	tools := Manifest{Module: "t", Version: "1", Tools: []string{"x"},
+		Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "go",
+			System: "arch", From: "cmd/t"}}}}
+	resolved, err := tools.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
+		Reference: ArtifactStoreScheme + "t/tools@" + aServiceDigest, Digest: aServiceDigest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if loads := resolved.Bundles[0].Loads; len(loads) != 1 || loads[0] != "t" {
+		t.Fatalf("a tools bundle nothing runs is no longer served: %v", loads)
+	}
+}
+
+// The account is created before anything is given to it. Its secrets are mesh-computed and so
+// placed before the module's own resources; given to a user the machine did not have yet, they were
+// refused on the first apply and the process started without them.
+func TestAModulesAccountComesBeforeWhatBelongsToIt(t *testing.T) {
+	out := composeTheService(t, Rendering{})
+	account, secret := indexOf(out, "svc.account"), indexOf(out, "svc."+NeedID("store"))
+	if account < 0 || secret < 0 {
+		t.Fatalf("the account or the secret is missing: %v", out)
+	}
+	if account > secret {
+		t.Fatalf("the secret owned by svc is written before svc exists: account at %d, secret at %d",
+			account, secret)
+	}
+	if owner := out[secret]["owner"]; owner != "svc" {
+		t.Errorf("the secret belongs to %v, not the account its process runs as", owner)
+	}
+}
+
+// The process is the module's program; its preparation is the same program asked to prepare, as a
+// step before it — with the same account and environment, and handing nothing over.
+func TestAProcessIsPreparedByItsOwnProgram(t *testing.T) {
+	out := composeTheService(t, Rendering{})
+	step, process := indexOf(out, "svc.service-prepare"), indexOf(out, "svc.service")
+	if step < 0 || process < 0 || step > process {
+		t.Fatalf("the preparation is not a step before the process (%d, %d): %v", step, process, out)
+	}
+	s := out[step]
+	if s["type"] != "process" || s["run-once"] != true || s["name"] != "svc-prepare" {
+		t.Errorf("the preparation is not a run-once process: %v", s)
+	}
+	if run, _ := json.Marshal(s["run"]); string(run) != `["./svc","prepare"]` {
+		t.Errorf("the preparation runs %s", run)
+	}
+	if s["user"] != "svc" || s["source"] != out[process]["source"] {
+		t.Errorf("the preparation does not run the same bundle as the same account: %v", s)
+	}
+	if env, _ := s["env"].(map[string]any); env["SVC_STORE_FILE"] == nil {
+		t.Errorf("the preparation is not given the process's environment: %v", s["env"])
+	}
+	if _, has := s["replaces"]; has {
+		t.Errorf("the preparation would hand over what the process replaces: %v", s)
+	}
+	if _, has := s["args"]; has {
+		t.Errorf("the preparation carries a container's args: %v", s)
+	}
+}
+
+// What the process replaces is named as the host recorded it, `<module>.<id>`; unprefixed, the host
+// matches nothing and removes the container first, as before.
+func TestWhatAProcessReplacesIsNamedAsTheHostRecordedIt(t *testing.T) {
+	out := composeTheService(t, Rendering{})
+	p := out[indexOf(out, "svc.service")]
+	if got, _ := json.Marshal(p["replaces"]); string(got) != `["svc.server"]` {
+		t.Fatalf("the process replaces %s", got)
+	}
+}
+
+func TestWhatReplacesMayNameIsRefusedNearItsAuthor(t *testing.T) {
+	for what, resource := range map[string]string{
+		"a container":        `{"id":"c","type":"container","name":"c","image":"x@` + aServiceDigest + `","replaces":["old"]}`,
+		"a step":             `{"id":"p","type":"process","name":"p","run":["./p"],"run-once":true,"replaces":["old"]}`,
+		"something declared": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["p"]}`,
+		"another module's":   `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["other.old"]}`,
+		"not a list":         `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":"old"}`,
+	} {
+		raw := `{"module":"m","version":"1","resources":[` + resource + `]}`
+		if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "replace") {
+			t.Errorf("replaces on %s was accepted: %v", what, err)
+		}
+	}
+	ok := `{"module":"m","version":"1","resources":[{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["old"]}]}`
+	if _, err := ParseManifest([]byte(ok)); err != nil {
+		t.Errorf("a process replacing what its module no longer declares was refused: %v", err)
+	}
+}

internal/catalogue/backup_test.go

@@ -0,0 +1,72 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that
+// holds nothing does not have to.
+func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) {
+	bare, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"provides":[{"name":"postgres-database","scope":"mesh"}]}`))
+	if err != nil {
+		t.Fatalf("parsing refused it, and the rule is the check's: %v", err)
+	}
+	if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") {
+		t.Fatalf("a store with no backup passed the check: %v", problems)
+	}
+	backed, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"provides":[{"name":"postgres-database","scope":"mesh"}],
+		"resources":[{"id":"dumps","type":"directory","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if problems := CheckBackup(backed); len(problems) != 0 {
+		t.Fatalf("a store that contributes a backup was refused: %v", problems)
+	}
+	route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`))
+	if problems := CheckBackup(route); len(problems) != 0 {
+		t.Fatalf("a route was asked for a backup: %v", problems)
+	}
+}
+
+// A backup contribution names its module's own directories; one it does not declare is refused
+// where it is written rather than reaching the holder as the literal text.
+func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
+	if err == nil || !strings.Contains(err.Error(), "nowhere") {
+		t.Fatalf("a backup of an undeclared directory was accepted: %v", err)
+	}
+}
+
+// The holder receives every module's backup lines with each module's own directories filled, each
+// module's under a comment naming it — and a kind written in a shell's grammar is left untouched.
+func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
+	pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
+		"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	mail, err := ParseManifest([]byte(`{"module":"mail","version":"1",
+		"resources":[{"id":"spool","type":"directory","mode":"0700"}],
+		"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:spool}"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"# pg\nrun pg-dump-all\npath /srv/pg/dumps\n", "# mail\npath /var/lib/mail/spool\n"} {
+		if !strings.Contains(placed, want) {
+			t.Errorf("placed contributions lack %q:\n%s", want, placed)
+		}
+	}
+	if strings.Contains(placed, "${dir:") {
+		t.Errorf("a directory reached the holder unfilled:\n%s", placed)
+	}
+}

internal/catalogue/bound_into_files.go

@@ -21,8 +21,9 @@ import (
 // formats and gains no fields.
 //
 // **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is:
-// `at`, `as` and `from` are facts about any provision at all, and everything else comes from what
+// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are
-// the provider said it serves — whose keys are agreed by the requirement's name, not by this file.
+// facts about any provision at all, and everything else comes from what the provider said it
+// serves — whose keys are agreed by the requirement's name, not by this file.
 
 // bound is where a module says a value from one of its bindings belongs:
 // ${bound:<provision>.<key>}.
@@ -46,7 +47,7 @@ func boundUsed(content string) [][2]string {
 // Three facts the mesh states about any provision, plus whatever the provider said it serves. A
 // module may not reach a binding it does not have — the same boundary as a secret, for the same
 // reason.
-func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]string {
+func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) {
 	out := map[string]map[string]string{}
 	for _, want := range m.Wants() {
 		for i := range needs {
@@ -54,12 +55,20 @@ func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]str
 			if n.Name != want || n.For != m.Module {
 				continue
 			}
+			as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module))
 			values := map[string]string{
 				"at":   n.At,
 				"from": n.From,
-				"as":   ConsumerIdentity(node, IdentitySource(m.Slug, m.Module)),
+				"as":   as,
 			}
-			for key, value := range n.Serves {
+			// What the provider derives for this consumer rather than for all of them
+			// (novox/hq ADR 0201). Filled here, the one place a provision and the module
+			// requiring it are both in hand.
+			served, err := ServedTo(n.Serves, as)
+			if err != nil {
+				return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err)
+			}
+			for key, value := range served {
 				// The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000,
 				// which is what a float would write and what a connection string would refuse.
 				values[key] = plainly(value)
@@ -67,7 +76,7 @@ func knownFor(m Manifest, needs []Needed, node string) map[string]map[string]str
 			out[want] = values
 		}
 	}
-	return out
+	return out, nil
 }
 
 // withOwnNames adds a module's own composed names to what it may name from one binding:

internal/catalogue/bound_into_files_test.go

@@ -232,3 +232,34 @@ func TestTwoModulesOnOneNodeAreTwoIdentities(t *testing.T) {
 		t.Fatal("one module on two machines shares an identity")
 	}
 }
+
+// A machine's resolver configuration must name its resolver by address — it cannot resolve the name
+// of the thing it resolves names with (novox/hq ADR 0194). So a binding offers the providing
+// machine's private address beside its name, and only when the machine has one.
+func TestABindingOffersTheProvidersAddress(t *testing.T) {
+	consumer := func() Resolution {
+		return Resolution{
+			Node: "workstation",
+			Modules: []Manifest{{
+				Module:   "resolv-conf",
+				Requires: []string{"wildcard-resolution"},
+				Resources: []map[string]any{{
+					"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644",
+					"content": "nameserver ${bound:wildcard-resolution:address}\n",
+				}},
+			}},
+			Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", At: "anchor.internal", For: "resolv-conf"}},
+		}
+	}
+	out, err := consumer().Declaration(Rendering{Machines: map[string]string{"anchor.internal": "10.77.0.1"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := fileNamed(out, "resolv-conf.resolv")["content"]; got != "nameserver 10.77.0.1\n" {
+		t.Fatalf("the resolver is not named by its address: %q", got)
+	}
+	// A machine with no address yet: refused, never written with a blank where the address belongs.
+	if _, err := consumer().Declaration(Rendering{}); err == nil {
+		t.Fatal("a file naming an address the mesh does not have was composed")
+	}
+}

internal/catalogue/build.go

@@ -77,6 +77,7 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
 	// build compare equal.
 	out.Bundles = nil
 	if m.Build != nil {
+		run := runByAResource(m)
 		for _, a := range m.Build.Artifacts {
 			if a.Kind != ArtifactBundle {
 				continue
@@ -84,8 +85,13 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
 			made := by[a.Name]
 			// What the runtime loads: what the artifact said, else every entrypoint of a module
 			// that declares tools, else nothing (the field's own rule; see Artifact.Loads).
+			//
+			// **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213).
+			// A process the host runs is the module's service, not its tools: the controller declares
+			// the verbs it answers as `tools` and serves them itself, and its bundle would otherwise
+			// have been launched a second time by the node's runtime, as an MCP child it is not.
 			loads := append([]string(nil), a.Loads...)
-			if a.Loads == nil && len(m.Tools) > 0 {
+			if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] {
 				loads = append([]string(nil), a.Entrypoints...)
 				// A bundle compiled to a binary has no entrypoints: the binary is what it is, and what
 				// the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served
@@ -453,6 +459,18 @@ func BinaryOf(a Artifact) string {
 	return a.Name
 }
 
+// runByAResource is the artifacts one of a module's own resources names — a process that runs it,
+// a step, an archive that unpacks it — by name.
+func runByAResource(m Manifest) map[string]bool {
+	named := map[string]bool{}
+	for _, r := range m.Resources {
+		if a, ok := r["artifact"].(string); ok && a != "" {
+			named[a] = true
+		}
+	}
+	return named
+}
+
 // undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq
 // 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says
 // `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or
@@ -463,12 +481,7 @@ func undeliveredBundles(m Manifest) []string {
 	if m.Build == nil || m.Module == RuntimeModule {
 		return nil
 	}
-	named := map[string]bool{}
+	named := runByAResource(m)
-	for _, r := range m.Resources {
-		if a, ok := r["artifact"].(string); ok && a != "" {
-			named[a] = true
-		}
-	}
 	var problems []string
 	for _, a := range m.Build.Artifacts {
 		if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 {

internal/catalogue/catalogue_check_test.go

@@ -86,3 +86,36 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
 		t.Fatalf("%d value(s) name an installation:\n  %s", len(named), strings.Join(named, "\n  "))
 	}
 }
+
+// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module
+// providing a store contributes a backup to node-backup, so a store added is a store backed up.
+func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
+	root := catalogueRoot(t)
+	found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil || len(found) == 0 {
+		t.Fatalf("no manifests under %s: %v", root, err)
+	}
+	stores := 0
+	for _, p := range found {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			t.Fatalf("%s: %v", p, err)
+		}
+		m, err := ParseManifest(raw)
+		if err != nil {
+			continue // TestEveryCatalogueManifestParses says why
+		}
+		for _, o := range m.Provides {
+			if storeProvisions[o.Name] {
+				stores++
+				break
+			}
+		}
+		for _, problem := range CheckBackup(m) {
+			t.Error(problem)
+		}
+	}
+	if stores == 0 {
+		t.Fatal("no module in the catalogue provides a store, so this proved nothing")
+	}
+}

internal/catalogue/computed_test.go

@@ -159,7 +159,8 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
 	}
 }
 
-// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
+// reloading answers the runtime's trust as the networking module once did (novox/hq ADR 0102; it no
+// longer does, ADR 0222 — and beside the runtime's module it is refused, generated_collision_test): a file
 // written into, and the runtime reloaded on it.
 type reloading struct{}
 

internal/catalogue/consumer_into_serves.go

@@ -0,0 +1,311 @@
+package catalogue
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// What a provider derives for one consumer, said once in the provider's definition and delivered
+// to both ends (novox/hq ADR 0201, issue 124).
+//
+// A `serves` block is otherwise literal: the same values for every consumer. Where the provider
+// *names the resource* — a bucket, a database, a vhost — the name is derived from who is asking,
+// and before this the mesh had no channel for it. The provider recomputed it in its own code and
+// every consumer transcribed it into its own definition by hand, which is a copy of somebody
+// else's rule kept in agreement by nobody. One of three transcriptions was wrong for months.
+//
+// **The mesh learns no protocol here; it spells its own name in an alphabet it already knows.**
+// The only fact a served value may name is the identity the mesh itself minted for the consumer,
+// in one of two alphabets: as it was minted, and as a DNS label. Everything a provider wants
+// around it — a prefix, a suffix, a separator — it writes around the placeholder, because a
+// served value is a string.
+
+// consumerFact is `${consumer:<fact>}` or `${consumer:<fact>:<alphabet>}`.
+var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`)
+
+// consumerFacts are what a served value may name about the consumer it is being derived for.
+// One entry, deliberately: the identity is the one thing about a consumer the mesh itself chose,
+// so it is the one thing the mesh can hand to a provider without either end guessing.
+var consumerFacts = []string{"as"}
+
+// consumerAlphabets are the ways the mesh will write that identity. `dns` is the mesh's own
+// identifier with its separator written `-` instead of `_` — the whole of the difference between
+// the alphabet the mesh mints in and the one buckets, vhosts and hostnames accept.
+var consumerAlphabets = []string{"dns"}
+
+// ServedTo fills a provider's served values for one consumer.
+//
+// `as` is the identity the mesh minted for that consumer — the same string it is told to present
+// as a login. Values with no placeholder are returned exactly as they were, and a block with no
+// placeholder at all is returned unchanged, so this costs nothing for the providers that derive
+// nothing.
+//
+// Only strings carry placeholders. A number, a boolean or a nested object is a value the provider
+// stated outright, and is left alone.
+func ServedTo(serves map[string]any, as string) (map[string]any, error) {
+	if len(serves) == 0 {
+		return serves, nil
+	}
+	var out map[string]any
+	for _, key := range sortedAnyKeys(serves) {
+		text, ok := serves[key].(string)
+		if !ok || !strings.Contains(text, "${consumer:") {
+			continue
+		}
+		filled, err := consumerInto(text, as)
+		if err != nil {
+			return nil, fmt.Errorf("the value served as %q: %w", key, err)
+		}
+		if out == nil {
+			// Copied only once something actually changes: the caller's map is the manifest's,
+			// and a provider that derives nothing must not have it rewritten underneath it.
+			out = make(map[string]any, len(serves))
+			for k, v := range serves {
+				out[k] = v
+			}
+		}
+		out[key] = filled
+	}
+	if out == nil {
+		return serves, nil
+	}
+	return out, nil
+}
+
+// consumerInto replaces every `${consumer:…}` in one value.
+//
+// **A fact or an alphabet the mesh does not have is refused, not left standing.** Written through,
+// the literal `${consumer:as}` would reach a configuration file and be read as a bucket name,
+// failing somewhere that names neither the module nor the mesh — the same reasoning `${bound:…}`
+// is refused by (boundInto).
+func consumerInto(value, as string) (string, error) {
+	var failed error
+	out := consumerFact.ReplaceAllStringFunc(value, func(match string) string {
+		parts := consumerFact.FindStringSubmatch(match)
+		fact, alphabet := parts[1], parts[2]
+		if fact != "as" {
+			if failed == nil {
+				failed = fmt.Errorf(
+					"says %s, and the mesh states %s about a consumer", match, orNothing(consumerFacts))
+			}
+			return match
+		}
+		switch alphabet {
+		case "":
+			return as
+		case "dns":
+			return asDNSLabel(as)
+		default:
+			if failed == nil {
+				failed = fmt.Errorf(
+					"says %s, and the mesh writes an identity as %s", match, orNothing(consumerAlphabets))
+			}
+			return match
+		}
+	})
+	if failed != nil {
+		return "", failed
+	}
+	return out, nil
+}
+
+// asDNSLabel writes a minted identity as a DNS label.
+//
+// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
+// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
+// this is the separator and nothing else — no lower-casing of what is already lower case, no
+// truncation to a limit the identity is already inside, no padding of a name that is already long
+// enough. Each of those would be the mesh guessing at a rule it has not been given.
+func asDNSLabel(as string) string {
+	return strings.ReplaceAll(as, "_", "-")
+}
+
+// CheckServes refuses a `serves` block that names a consumer fact or an alphabet the mesh does not
+// have, when the definition is parsed rather than when a consumer is resolved.
+//
+// A provision nobody consumes yet still has its rule read: a definition that would be refused the
+// first time somebody required it is a definition that is wrong now.
+func CheckServes(m Manifest) []string {
+	var problems []string
+	for _, provision := range sortedServes(m.Serves) {
+		for _, key := range sortedAnyKeys(m.Serves[provision]) {
+			text, ok := m.Serves[provision][key].(string)
+			if !ok {
+				continue
+			}
+			// A probe identity, because what is checked is the shape of the statement and not
+			// what any consumer is called.
+			if _, err := consumerInto(text, "mesh_node_module"); err != nil {
+				problems = append(problems, fmt.Sprintf(
+					"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
+			}
+		}
+	}
+	return problems
+}
+
+func sortedServes(serves map[string]map[string]any) []string {
+	out := make([]string, 0, len(serves))
+	for k := range serves {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func sortedAnyKeys(values map[string]any) []string {
+	out := make([]string, 0, len(values))
+	for k := range values {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// derivedFor is what the provider on this machine derives for one consumer of one provision
+// (novox/hq ADR 0201).
+//
+// Settled first, then derived: an operator may set a prefix on what the provider serves and the
+// mesh still fills the consumer's half of it ([ADR 0174]). Only the keys that actually name the
+// consumer are returned — the rest of a `serves` block is the same for every consumer and is
+// already in the provider's own definition, so repeating it here would be a second copy to go
+// stale.
+//
+// The first module in the resolved order that says it serves the provision answers, which is the
+// choice servedOnThisMachine makes for the consumer's half. Nothing serving it on this machine is
+// not an error: a contribution can reach a machine whose provider is a record or an adapter, and
+// then there is nothing derived to tell.
+func (r Resolution) derivedFor(provision, as, consumer, local string, settings SettingsBy) (map[string]any, error) {
+	for _, m := range r.Modules {
+		serves, said := m.Serves[provision]
+		if !said {
+			continue
+		}
+		var names map[string]any
+		for key, value := range serves {
+			if text, ok := value.(string); ok && strings.Contains(text, "${consumer:") {
+				if names == nil {
+					names = map[string]any{}
+				}
+				names[key] = value
+			}
+		}
+		if names == nil {
+			return nil, nil
+		}
+		// **A consumer that keeps several holders of this provision is refused** — this is issue
+		// 124's own failure one case to the side, and it would be just as quiet.
+		//
+		// Each holder gets its own login, `…_<local>` (ADR 0094), and a provider derives from the
+		// login, so it would make one resource per holder. The consumer's side has no such
+		// dimension: one binding file per provision, one `${bound:<provision>:<key>}`, both
+		// derived from the un-suffixed identity. So the provider would create the holder's
+		// resource and the consumer would be configured against a name nothing made — it would
+		// authenticate successfully and be refused on every object, which reads like a credential
+		// fault and is not one.
+		//
+		// Lifting this means giving the consumer's side a local dimension. That is a decision,
+		// not an omission, and until it is taken the mesh says so rather than guessing.
+		if local != "" {
+			return nil, fmt.Errorf(
+				"%s keeps several holders of %s (this one is %q), and %s derives %s for each "+
+					"consumer from the login the mesh minted. Each holder has its own login, and a "+
+					"consumer is told one value per requirement — so the two ends would name "+
+					"different things and nothing would compare them (novox/hq ADR 0201)",
+				consumer, local, provision, m.Module, orNothing(sortedAnyKeys(names)))
+		}
+		settled, err := Settle(names, settings[m.Module])
+		if err != nil {
+			return nil, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
+		}
+		derived, err := ServedTo(settled, as)
+		if err != nil {
+			return nil, fmt.Errorf("%s serving %s to %s: %w", m.Module, provision, as, err)
+		}
+		return derived, nil
+	}
+	return nil, nil
+}
+
+// notTranscribed refuses a consumer's file that writes out the value its provider derives for it,
+// instead of asking for it (novox/hq ADR 0201, issue 124).
+//
+// **What would have caught the one wrong instance.** The object store's three consumers each wrote
+// their bucket into their own configuration by hand. One of them named a predecessor's bucket, and
+// nothing compared it to what the provider would actually create: the module would have
+// authenticated successfully and been refused on every object, which reads like a credential fault
+// and is not one. It looked authoritative for months.
+//
+// The test is exact and costs one string search: a definition whose file already contains the
+// value the mesh is about to derive for it has written down somebody else's rule. It cannot be a
+// coincidence — a derived value carries the identity the mesh minted for this very consumer on
+// this very machine, which nothing else would spell out — and it cannot be checked afterwards,
+// because after substitution every consumer's file contains it legitimately.
+//
+// Only values that actually name the consumer are judged. A provider that serves a constant under
+// the same key serves the same constant to everyone, and a consumer repeating it is redundant
+// rather than wrong.
+func notTranscribed(resource map[string]any, known map[string]map[string]string, module string) error {
+	if fmt.Sprint(resource["type"]) != "file" {
+		return nil
+	}
+	content, ok := resource["content"].(string)
+	if !ok || content == "" {
+		return nil
+	}
+	for _, provision := range sortedKnown(known) {
+		values := known[provision]
+		identity := values["as"]
+		if identity == "" {
+			continue
+		}
+		for _, key := range sortedStringKeys(values) {
+			if key == "as" {
+				// The login is not derived from itself, and a consumer that must present it in a
+				// connection string legitimately has it from `${bound:…}` — which is what it will
+				// be after substitution, so this would judge the substitution, not the module.
+				continue
+			}
+			value := values[key]
+			if value == "" || !namesTheConsumer(value, identity) {
+				continue
+			}
+			if !strings.Contains(content, value) {
+				continue
+			}
+			return fmt.Errorf(
+				"%s writes %q into %v, and that is exactly what %s derives for it — a definition "+
+					"keeping its own copy of somebody else's naming rule is one that can disagree "+
+					"with it, silently. Say ${bound:%s:%s} and be told",
+				module, value, resource["id"], provision, provision, key)
+		}
+	}
+	return nil
+}
+
+// namesTheConsumer is whether a derived value was built from this consumer's identity — in the
+// alphabet it was minted in, or as a DNS label. A value that does not contain it was not derived
+// from it, whatever else it may be.
+func namesTheConsumer(value, identity string) bool {
+	return strings.Contains(value, identity) || strings.Contains(value, asDNSLabel(identity))
+}
+
+func sortedKnown(known map[string]map[string]string) []string {
+	out := make([]string, 0, len(known))
+	for k := range known {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func sortedStringKeys(values map[string]string) []string {
+	out := make([]string, 0, len(values))
+	for k := range values {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}

internal/catalogue/contribution_dependencies_test.go

@@ -0,0 +1,121 @@
+package catalogue
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Defends novox/hq ADR 0210 §3: a contribution is a dependency on the seat that receives it.
+
+func TestAContributionDependsOnTheSeatThatPlacesIt(t *testing.T) {
+	env := mod("theme", nil, nil, nil)
+	env.Environment = &Environment{Variables: map[string]string{"GTK_THEME": "Adwaita:dark"}}
+	path := mod("toolchain", nil, nil, nil)
+	path.Environment = &Environment{Path: []PathEntry{{Entry: "/opt/x/bin"}}}
+	shell := mod("prompt", nil, nil, nil)
+	shell.Shell = []ShellCode{{For: "zsh", Slot: "first", Code: "true"}}
+	session := mod("wallpaper", nil, nil, nil)
+	session.Shell = []ShellCode{{For: "xinitrc", Slot: "normal", Code: "true"}}
+	resources := mod("bar", nil, nil, nil)
+	resources.Shell = []ShellCode{{For: "xresources", Slot: "normal", Code: "x: y"}}
+	empty := mod("nothing", nil, nil, nil)
+	empty.Environment = &Environment{}
+
+	cases := map[string]struct {
+		m    Manifest
+		want []string
+	}{
+		"a variable":                {env, []string{EnvironmentSeat}},
+		"a path entry":              {path, []string{EnvironmentSeat}},
+		"shell code":                {shell, []string{LoginShellSeat}},
+		"the session's start":       {session, []string{DisplayServerSeat}},
+		"the session's X resources": {resources, []string{DisplayServerSeat}},
+		"an empty environment":      {empty, nil},
+	}
+	for name, c := range cases {
+		got := DependsOn(c.m)
+		if len(got) == 0 {
+			got = nil
+		}
+		if !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%s depends on %v, want %v", name, got, c.want)
+		}
+	}
+}
+
+func TestAContributionIsMetByAHolderOnTheNodeAndRefusedWithout(t *testing.T) {
+	holder := mod("node-env", nil, nil, nil, Claim{Name: EnvironmentSeat})
+	contributor := mod("theme", nil, nil, nil)
+	contributor.Environment = &Environment{Variables: map[string]string{"GTK_THEME": "Adwaita:dark"}}
+	catalogue := map[string]Manifest{"node-env": holder, "theme": contributor}
+
+	if _, err := AssignRefusal(catalogue, "laptop", []string{"node-env"}, []string{"theme"}); err != nil {
+		t.Fatalf("a contributor beside the holder is refused: %v", err)
+	}
+	_, err := AssignRefusal(catalogue, "laptop", nil, []string{"theme"})
+	if err == nil {
+		t.Fatal("a contributor on a node without the holder was accepted, and its contribution would be written nowhere")
+	}
+	if !strings.Contains(err.Error(), EnvironmentSeat) || !strings.Contains(err.Error(), "node-env") {
+		t.Errorf("the refusal names neither the seat nor its holder: %v", err)
+	}
+	if _, err := AssignRefusal(catalogue, "laptop", nil, []string{"theme", "node-env"}); err != nil {
+		t.Errorf("the contributor and the holder assigned together are refused: %v", err)
+	}
+}
+
+func TestAHolderMeetsItsOwnContribution(t *testing.T) {
+	// The display server's module contributes nothing to its own seat today, but the zsh module's
+	// environment contributions do go to another seat: a claim meets only the seat it names.
+	zsh := mod("zsh", nil, nil, nil, Claim{Name: LoginShellSeat})
+	zsh.Shell = []ShellCode{{For: "zsh", Slot: "normal", Code: "true"}}
+	zsh.Environment = &Environment{Variables: map[string]string{"EDITOR": "vim"}}
+	catalogue := map[string]Manifest{"zsh": zsh,
+		"node-env": mod("node-env", nil, nil, nil, Claim{Name: EnvironmentSeat})}
+	unheld := UnheldDependencies(catalogue, "laptop", []Manifest{zsh}, nil)
+	if len(unheld) != 1 || unheld[0].Seat != EnvironmentSeat {
+		t.Errorf("zsh alone: unheld %v, want only %s (its shell code is its own seat's)", unheld, EnvironmentSeat)
+	}
+}
+
+func TestTwoModulesDeclaringOnePackageAreRefusedBeforeAnythingIsRecorded(t *testing.T) {
+	pacman := withResources(mod("pacman", nil, nil, nil, Claim{Name: PackageManagerSeat}),
+		res("package", "pacman-contrib"))
+	bar := withResources(mod("bar", nil, nil, nil), res("package", "bar"), res("package", "pacman-contrib"))
+	other := withResources(mod("other", nil, nil, nil), res("package", "other"))
+	catalogue := map[string]Manifest{"pacman": pacman, "bar": bar, "other": other}
+
+	err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"bar"})
+	if err == nil || !strings.Contains(err.Error(), "pacman-contrib") || !strings.Contains(err.Error(), "bar") {
+		t.Fatalf("the second owner of a package was not refused by name: %v", err)
+	}
+	if err := CollisionRefusal(catalogue, "laptop", []string{"pacman"}, []string{"other"}); err != nil {
+		t.Errorf("a module declaring nothing shared is refused: %v", err)
+	}
+	// A collision already on the node is status's, not a reason to refuse an unrelated assignment.
+	if err := CollisionRefusal(catalogue, "laptop", []string{"pacman", "bar"}, []string{"other"}); err != nil {
+		t.Errorf("an unrelated assignment is refused for a collision already there: %v", err)
+	}
+}
+
+func TestCodeForAPowerMomentDependsOnThePowerSeat(t *testing.T) {
+	for _, moment := range powerMoments {
+		m := mod("laptop", nil, nil, nil)
+		m.Shell = []ShellCode{{For: moment, Slot: "normal", Code: "true"}}
+		if got := DependsOn(m); !reflect.DeepEqual(got, []string{PowerSeat}) {
+			t.Errorf("code for %s depends on %v, want %s", moment, got, PowerSeat)
+		}
+		if p := m.shellProblems(); len(p) != 0 {
+			t.Errorf("code for %s is refused: %v", moment, p)
+		}
+	}
+	m := mod("laptop", nil, nil, nil)
+	m.Shell = []ShellCode{{For: "after-lunch", Slot: "normal", Code: "true"}}
+	if p := m.shellProblems(); len(p) == 0 {
+		t.Error("code for a moment that does not exist was accepted")
+	}
+	if s, ok := SeatNamed(PowerSeat); !ok || s.Scope != ScopeNode {
+		t.Errorf("%s is not a node seat of the mesh's own: %+v %v", PowerSeat, s, ok)
+	}
+}

internal/catalogue/controller_is_a_process_test.go

@@ -0,0 +1,131 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 213: the controller is a Go program and was the one piece of the mesh's own Go
+// code still shipped as an image (ADR 0188 §1). Its own manifest, composed for the machine that runs
+// it, is a Go bundle run by the host as a process — and no container.
+func TestTheControllerIsAProcessAndNoContainer(t *testing.T) {
+	raw, err := os.ReadFile("../../module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := ParseManifest(raw)
+	if err != nil {
+		t.Fatalf("the controller's own manifest does not parse:\n%v", err)
+	}
+	if m.Build == nil || len(m.Build.Artifacts) != 1 {
+		t.Fatalf("the controller builds %+v; it is one bundle", m.Build)
+	}
+	a := m.Build.Artifacts[0]
+	if a.Kind != ArtifactBundle || a.Language != "go" || a.System == "" || BinaryOf(a) != "mesh-controller" {
+		t.Fatalf("the controller's artifact is %+v, not a Go bundle naming its system and binary", a)
+	}
+	for _, c := range m.Capabilities {
+		if c == "container-runtime" {
+			t.Error("the controller still requires a container runtime on its machine")
+		}
+	}
+
+	digest := "sha256:" + strings.Repeat("c", 64)
+	control, err := m.Resolve([]Built{{Name: a.Name, Kind: ArtifactBundle,
+		Reference: ArtifactStoreScheme + "mesh-controller/" + a.Name + "@" + digest, Digest: digest}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The node's runtime does not launch it: it serves its seat's verbs itself.
+	if loads := control.Bundles[0].Loads; len(loads) != 0 {
+		t.Errorf("the node's runtime would launch the controller as a tools bundle: %v", loads)
+	}
+
+	needed := map[string]map[string]string{"mesh-controller": {}}
+	for name := range m.OwnSecrets {
+		needed["mesh-controller"][name] = "sealed-" + name
+	}
+	out, err := Resolution{Node: "anchor", Modules: []Manifest{control}}.Declaration(Rendering{
+		Needed: needed, ArtifactStore: "anchor.internal:5100",
+		Seats: map[string]map[int]int{"mesh-store": {5432: 6852}},
+	})
+	if err != nil {
+		t.Fatalf("the controller does not compose: %v", err)
+	}
+
+	var process, step map[string]any
+	account, firstSecret := -1, -1
+	for i, r := range out {
+		switch {
+		case r["type"] == "container":
+			t.Errorf("the controller's declaration still runs a container: %v", r)
+		case r["id"] == "mesh-controller.controller":
+			process = r
+		case r["id"] == "mesh-controller.controller-prepare":
+			step = r
+			if process != nil {
+				t.Error("the controller's preparation is placed after the process it prepares for")
+			}
+		case r["type"] == "user" && r["name"] == "mesh-controller":
+			account = i
+		case strings.HasPrefix(fmt.Sprint(r["id"]), "mesh-controller.needs-") && firstSecret < 0:
+			firstSecret = i
+		}
+	}
+	if process == nil {
+		t.Fatalf("the controller's process is not in its declaration: %v", out)
+	}
+	if run, _ := json.Marshal(process["run"]); string(run) != `["./mesh-controller","serve"]` {
+		t.Errorf("the controller is run as %s, not its own bundle's binary", run)
+	}
+	if process["source"] != "anchor.internal:5100/mesh-controller/"+a.Name+"@"+digest || process["digest"] != digest {
+		t.Errorf("the controller's bundle is fetched from %v (%v)", process["source"], process["digest"])
+	}
+	// The user: an account the host declares, which owns what the process reads.
+	if process["user"] != "mesh-controller" || account < 0 {
+		t.Errorf("the controller runs as %v, and the account declared is at %d", process["user"], account)
+	}
+	if firstSecret >= 0 && account > firstSecret {
+		t.Error("the controller's secrets are written before the account they belong to exists")
+	}
+	for _, r := range out {
+		if strings.HasPrefix(fmt.Sprint(r["id"]), "mesh-controller.needs-") && r["owner"] != "mesh-controller" {
+			t.Errorf("%v belongs to %v, which the controller's process cannot read", r["id"], r["owner"])
+		}
+	}
+	if dir := fileNamed(out, "mesh-controller.mesh-state"); dir == nil || dir["owner"] != "mesh-controller" {
+		t.Errorf("the controller's state directory is not its account's to enter: %v", dir)
+	}
+	// Each mount became a path the process reads: nothing it is told is a path inside a container.
+	state := fmt.Sprint(fileNamed(out, "mesh-controller.mesh-state")["path"])
+	env, _ := process["env"].(map[string]any)
+	for key, value := range env {
+		v := fmt.Sprint(value)
+		if strings.HasPrefix(v, "/run/secrets") || strings.HasPrefix(v, "/broker-tls") {
+			t.Errorf("%s=%s is a path inside the container the controller no longer runs in", key, v)
+		}
+		if strings.HasSuffix(key, "_FILE") && !strings.HasPrefix(v, state+"/") {
+			t.Errorf("%s=%s is not one of the files the mesh places for it", key, v)
+		}
+	}
+	if env["MESH_BROKER_CERTIFICATE"] != "/var/lib/mesh-broker-tls/tls.crt" {
+		t.Errorf("the controller reads the broker's certificate from %v", env["MESH_BROKER_CERTIFICATE"])
+	}
+	if env["MESH_STORE_INVENTORY_PORT"] != "6852" {
+		t.Errorf("the controller is told the store is on %v; the node put it on 6852", env["MESH_STORE_INVENTORY_PORT"])
+	}
+	// The handover: the container it ran as goes only once this is running.
+	if got, _ := json.Marshal(process["replaces"]); string(got) != `["mesh-controller.server"]` {
+		t.Errorf("the controller's process replaces %s, not the container it ran as", got)
+	}
+	// And its state is prepared first, by the same program as the same account.
+	if step == nil || step["run-once"] != true || step["user"] != "mesh-controller" {
+		t.Fatalf("the controller's preparation is %v", step)
+	}
+	if run, _ := json.Marshal(step["run"]); string(run) != `["./mesh-controller","prepare"]` {
+		t.Errorf("the controller's preparation runs %s", run)
+	}
+}

internal/catalogue/declaration.go

@@ -155,6 +155,15 @@ type Rendering struct {
 	// standing beside the machines and looking as real as they do.
 	Machines map[string]string
 
+	// Zones is every zone a module in the mesh answers itself, where it is answered (novox/hq ADR
+	// 0199): the mesh's resolver forwards each one there.
+	Zones []ZoneAt
+
+	// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
+	// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
+	// file lists: every holder of the mesh's resolver, this machine first if it is one.
+	Holders map[string]map[string]string
+
 	Settings   SettingsBy
 	Generators map[string]Generator
 	// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -197,6 +206,11 @@ type Rendering struct {
 	// stored (novox/hq 04-ISSUES/102).
 	ArtifactStore string
 
+	// SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked
+	// about (host:port), by seat: what ${seat:<seat>:reach} answers with (novox/hq ADR 0222). Absent
+	// when no holder is reachable yet; see seat_into.go for which seats are answered.
+	SeatReach map[string]string
+
 	// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
 	// with an address — before references were kept without one — from an image a module runs
 	// straight from a public registry.
@@ -415,6 +429,18 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 	filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
 		with.OutwardLinks, with.TunnelInterface)
 
+	// **A variable two modules set is refused whether or not anything places it** (novox/hq ADR
+	// 0203 §5): the account has one environment, and a machine whose holder arrives later should not
+	// be the moment two modules are found to disagree about it.
+	if err := variablesSetOnce(r.Modules); err != nil {
+		return nil, err
+	}
+
+	generated, err := r.generatedHere(with)
+	if err != nil {
+		return nil, err
+	}
+
 	var out []map[string]any
 	for _, m := range r.Modules {
 		if with.Adopted && m.Filtering != nil {
@@ -603,14 +629,14 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 					// and nothing would say so.
 					continue
 				}
-				first = append(first, map[string]any{
+				first = append(first, ownedBy(r.provisionsAs(m), map[string]any{
 					// One file per holder — the consumer's module with its local name after it
 					// where it keeps several (ADR 0094); the lab found two files with one id.
 					"id":     GrantID(to, g.Consumer+"."+holderAs(g.From, g.Local)),
 					"type":   "file",
 					"path":   grantPath(m.Grants[to], g.Consumer, holderAs(g.From, g.Local)),
 					"sealed": g.Sealed,
-				})
+				}))
 			}
 		}
 		for _, to := range sortedKeys(m.Binds) {
@@ -645,7 +671,16 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			if err != nil {
 				return nil, err
 			}
-			file, err := boundFile(*found, m.Binds[to], ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module)), own)
+			as := ConsumerIdentity(r.Node, IdentitySource(m.Slug, m.Module))
+			// What the provider derives for THIS consumer, filled here where the consumer is
+			// known (novox/hq ADR 0201). The same fill knownFor does below, so the binding file
+			// and the module's `${bound:…}` substitutions cannot say different things.
+			told := *found
+			told.Serves, err = ServedTo(told.Serves, as)
+			if err != nil {
+				return nil, fmt.Errorf("%s is told about %s: %w", m.Module, to, err)
+			}
+			file, err := boundFile(told, m.Binds[to], as, own)
 			if err != nil {
 				return nil, err
 			}
@@ -672,28 +707,27 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			first = append(first, file)
 		}
 		if m.Computed != "" {
-			generator, known := with.Generators[m.Computed]
+			mine, part := generated[m.Module]
-			if !known {
-				return nil, fmt.Errorf(
-					"%s says its resources are computed by %q, and this control plane has no %q",
-					m.Module, m.Computed, m.Computed)
-			}
-			generated, part, err := generator.Resources(r.Node)
-			if err != nil {
-				return nil, err
-			}
 			if !part {
 				// Assigned, and not yet part of what this generates. Nothing to put on the
 				// machine, which is different from an error: a node given the network module
 				// before it has an address is in exactly that state, briefly.
 				continue
 			}
-			resources = generated
+			resources = mine
 		}
 
 		// Now, and not before: a module whose resources are computed replaces them wholesale, and
 		// merging earlier would throw away the files it still needs.
-		resources = append(append([]map[string]any{}, first...), resources...)
+		//
+		// **Except the module's own accounts, which go before even those** (novox/hq issue 213). What
+		// the mesh computes may belong to one: a module whose code runs as an account it declares has
+		// its secrets written owned by that account, and a file given to a user the machine does not
+		// have yet fails — so on the first apply the secrets were refused, the process started without
+		// them, and the second apply healed it, which is the fault the paragraph above describes.
+		// An account depends on nothing the mesh computes.
+		accounts, rest := accountsFirst(resources)
+		resources = append(append(accounts, first...), rest...)
 
 		// No container is given the mesh's names (novox/hq ADR 0148). It used to be: every
 		// container got the whole roster as `--add-host` entries at creation, and a name that
@@ -711,7 +745,10 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			return nil, err
 		}
 		// And what its bindings say, for the half of a connection that is not secret.
-		known := knownFor(m, r.Needs, r.Node)
+		known, err := knownFor(m, r.Needs, r.Node)
+		if err != nil {
+			return nil, err
+		}
 		// A requirement answered on this same machine is not in r.Needs — its binding file is
 		// written from `here` (above) — and so `${bound:…}` could not name it, though the file
 		// beside it said the same facts. Filled from the same answer, so the two cannot disagree.
@@ -728,10 +765,24 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			}
 			local := *answered
 			local.For = m.Module
-			for provision, values := range knownFor(m, []Needed{local}, r.Node) {
+			here, err := knownFor(m, []Needed{local}, r.Node)
+			if err != nil {
+				return nil, err
+			}
+			for provision, values := range here {
 				known[provision] = values
 			}
 		}
+		// And the providing machine's private address, beside its name (novox/hq ADR 0194). A name is
+		// what nearly every consumer wants; the one that cannot use it is a machine's resolver
+		// configuration, which must reach the resolver before it can resolve anything — the resolver's
+		// own name included. Absent when the machine has no address yet, so a file naming it is refused
+		// rather than written with a blank where an address belongs.
+		for _, values := range known {
+			if address := with.Machines[values["at"]]; address != "" {
+				values["address"] = address
+			}
+		}
 		// And what the module is called through each requirement it contributes to (novox/hq
 		// 04-ISSUES/122) — the same composition its binding file carries.
 		for provision, values := range known {
@@ -753,6 +804,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		// And the machine underneath, which no binding of its own can tell it.
 		thisMachine := machineFacts(r, with.Names, with.MeshRange)
 
+		// **A definition that already holds the answer transcribed it** (novox/hq ADR 0201).
+		// Judged over what the module itself declares, and before anything is substituted: the
+		// mesh's own generated files — the binding, the contributions — legitimately carry the
+		// derived value, and after substitution so does every consumer's file, so this is the one
+		// moment the two can be told apart.
+		for _, own := range m.Resources {
+			if err := notTranscribed(own, known, m.Module); err != nil {
+				return nil, err
+			}
+		}
+
 		// Which of this module's files carry a secret, for the rule that a container may not read
 		// one of them as its environment without saying so (ADR 0086, issue 041).
 		secretFiles := secretFilesOf(resources)
@@ -854,6 +916,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 				return nil, err
 			}
 			publishedOn(copied, m.Module, with)
+			// The account's environment and every module's shell code, where this module holds the
+			// seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on
+			// the node, as the jails are, and **last of every placeholder pass**: shell code is a
+			// shell's own syntax, full of `${…}` no pass above should ever be shown.
+			if err := contributionsInto(copied, m, r.Modules, thisMachine, with); err != nil {
+				return nil, err
+			}
 			copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
 			// A service saying what it reflects names resources within its own module, so those
 			// are prefixed too or they would point at nothing.
@@ -872,6 +941,21 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 			if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
 				copied["reload-on"] = renamed
 			}
+			// And which of its module's containers a scheduled step holds still (novox/hq ADR 0189).
+			// **The loudest of the three when it is missed.** An unprefixed `restart-on` matches
+			// nothing and a service quietly never restarts; an unprefixed `while-stopped` names a
+			// container the declaration does not contain, and the host refuses the whole
+			// declaration — so the machine takes nothing at all, for every push, until this is
+			// right. That is what it did on the control node (2026-10-04).
+			if renamed := reflectsRenamed(m.Module, resource[WhileStopped]); renamed != nil {
+				copied[WhileStopped] = renamed
+			}
+			// And what a process replaces (novox/hq issue 213): a resource of this module's that it
+			// no longer declares, named as the host recorded it, or the host hands nothing over and
+			// removes it first.
+			if renamed := reflectsRenamed(m.Module, resource["replaces"]); renamed != nil {
+				copied["replaces"] = renamed
+			}
 			// **What reads one of this module's own secrets is restarted when it changes** (novox/hq
 			// issue 203, issue 206). A credential is re-issued by the mesh, and a container that
 			// mounted the old file keeps the old one open: the build machine ran for an hour on a
@@ -901,7 +985,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
 		// plane's; making a name resolve is the module's software. Emitted as ordinary files under
 		// this module's name, so they are applied, reported and removed exactly as anything else
 		// it declares.
-		given, err := FactsInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix)
+		given, err := FactsFrom(m, r, with)
 		if err != nil {
 			return nil, err
 		}
@@ -1161,6 +1245,41 @@ type Contribution struct {
 	// requirement's name — everything providing `reverse-proxy` understands the same shape, which
 	// is what makes swapping one for another cost nothing.
 	Values map[string]any `json:"values"`
+	// Derived is what this provider's own definition said it derives for this consumer, already
+	// derived (novox/hq ADR 0201).
+	//
+	// **The provider is told, rather than recomputing it.** A served value may name the consumer's
+	// identity — a bucket named for who is asking, a database prefixed with it — and before this
+	// the rule lived twice: once in the provisioner's code, once transcribed into every consumer's
+	// definition. The mesh fills the provider's own statement here and delivers the same filled
+	// value to the consumer, so the two cannot disagree: there is no second computation to
+	// disagree with.
+	//
+	// Only the keys that are per-consumer. The rest of what the provider serves is the same for
+	// everyone and is in its own definition, where it already is.
+	Derived map[string]any `json:"derived,omitempty"`
+}
+
+// provisionsAs is the account that reads what the mesh writes for this provider: the one secret
+// per consumer it must open to set that consumer's password (novox/hq issue 225).
+//
+// **A root-owned 0600 file is one that process cannot read**, which is the same sentence already
+// written above for a module's own secrets — and the grant secret is the other kind of secret
+// the mesh writes for a module, so it is the same rule.
+//
+// Which account depends on where the module's code runs. A module whose code is a bundle is run
+// by the node's tool runtime, as the node's account ([ADR 0198](0198)); one still in a container
+// is whatever it declares as its secrets owner. Nothing names these paths, so the rule that
+// claims a bundle's other files by the words that name them (givenTo) cannot reach them: the
+// harness composes a grant secret's path from the contributions file, not from a word.
+//
+// Empty is root, which is what it was and what a module with no bundle and no declared owner
+// still wants.
+func (r Resolution) provisionsAs(m Manifest) string {
+	if len(m.Bundles) > 0 && r.Account != "" {
+		return r.Account
+	}
+	return m.SecretsOwner
 }
 
 // grantPath is where one consumer's sealed credential lands on the providing machine.
@@ -1252,12 +1371,17 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 			// told about it and withdraws the login on its next pass.
 			continue
 		}
+		as := holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local)
+		derived, err := r.derivedFor(g.Provision, as, g.From, g.Local, settings)
+		if err != nil {
+			return nil, err
+		}
 		out[g.Provision] = append(out[g.Provision], Contribution{
-			From: g.From, Node: g.Consumer, At: g.At, Values: g.Values,
+			From: g.From, Node: g.Consumer, At: g.At, Values: g.Values, Derived: derived,
 			// One holder per local name: the identity the consumer is known by, and the local name
 			// after it where the module keeps several (ADR 0094). Not a login any backend checks —
 			// a secret is not a login — so the identity limit does not apply to the suffix.
-			As:     holderAs(ConsumerIdentity(g.Consumer, IdentitySource(g.Slug, g.From)), g.Local),
+			As:     as,
 			Secret: grantPath(directories[g.Provision], g.Consumer, holderAs(g.From, g.Local)),
 		})
 		if granted[g.Provision] == nil {
@@ -2010,12 +2134,18 @@ func preparationTarget(m Manifest) string {
 		return ""
 	}
 	for _, r := range m.Resources {
-		if fmt.Sprint(r["type"]) != "container" || !ownArtifact(r, m.Module) {
+		// A container, or a process the host runs from a bundle the module built (novox/hq issue
+		// 213): the same program in the same context, hosted as a unit rather than a container.
+		kind := fmt.Sprint(r["type"])
+		if (kind != "container" && kind != "process") || !ownArtifact(r, m.Module) {
 			continue
 		}
 		if once, _ := r["run-once"].(bool); once {
 			continue
 		}
+		if r["schedule"] != nil {
+			continue
+		}
 		return fmt.Sprint(r["id"])
 	}
 	return ""
@@ -2029,7 +2159,10 @@ func ownArtifact(resource map[string]any, module string) bool {
 		return true
 	}
 	image, _ := resource["image"].(string)
-	return strings.HasPrefix(image, ArtifactStoreScheme+module+"/")
+	// A process or an archive carries what was built as its source (novox/hq issue 213).
+	source, _ := resource["source"].(string)
+	return strings.HasPrefix(image, ArtifactStoreScheme+module+"/") ||
+		strings.HasPrefix(source, ArtifactStoreScheme+module+"/")
 }
 
 // prepared is the module's own resource as the step that prepares its state: the same image, the same
@@ -2051,7 +2184,19 @@ func prepared(from map[string]any) map[string]any {
 	step["id"] = fmt.Sprint(from["id"]) + "-prepare"
 	step["name"] = fmt.Sprint(from["name"]) + "-prepare"
 	step["run-once"] = true
-	step["args"] = []any{PreparationArgument}
+	if fmt.Sprint(from["type"]) == "process" {
+		// A process says its whole command: the program, then its arguments. The step is the same
+		// program asked to prepare (novox/hq issue 213). It replaces nothing — what the process
+		// replaces is handed over to the process, never to the step that runs before it — and a
+		// step is not restarted, it runs again when what it reads changed, which `restart-on` says.
+		run := stringsIn(from["run"])
+		if len(run) > 0 {
+			step["run"] = []any{run[0], PreparationArgument}
+		}
+		delete(step, "replaces")
+	} else {
+		step["args"] = []any{PreparationArgument}
+	}
 	delete(step, "ports")
 	delete(step, "ip")
 	delete(step, "schedule")
@@ -2196,3 +2341,62 @@ func withRestartOn(have any, add []string) []any {
 	}
 	return out
 }
+
+// accountsFirst splits a module's resources into its accounts and everything else, each in the order
+// written.
+func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any) {
+	for _, r := range resources {
+		if fmt.Sprint(r["type"]) == "user" {
+			accounts = append(accounts, r)
+			continue
+		}
+		rest = append(rest, r)
+	}
+	return accounts, rest
+}
+
+// generatedHere is what each computed module's generator answers for this machine, by module —
+// absent for a module whose machine is not yet part of what it generates — held to the rule every
+// module is held to: no two modules on a machine declare one path, unit, name or package (novox/hq
+// issue 190, step 5; ADR 0222).
+//
+// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
+// resources it will declare — they exist only once its generator has answered for this machine,
+// here — so a generated resource writing into another module's file was never seen. That is how
+// the private network came to declare the container runtime's daemon file and service beside the
+// runtime's own module. Asked once, so what is checked is exactly what is declared.
+func (r Resolution) generatedHere(with Rendering) (map[string][]map[string]any, error) {
+	generated := map[string][]map[string]any{}
+	placed := make([]Manifest, 0, len(r.Modules))
+	for _, m := range r.Modules {
+		if m.Computed == "" {
+			placed = append(placed, m)
+			continue
+		}
+		generator, known := with.Generators[m.Computed]
+		if !known {
+			return nil, fmt.Errorf(
+				"%s says its resources are computed by %q, and this control plane has no %q",
+				m.Module, m.Computed, m.Computed)
+		}
+		resources, part, err := generator.Resources(r.Node)
+		if err != nil {
+			return nil, err
+		}
+		computed := m
+		computed.Resources = nil
+		if part {
+			generated[m.Module] = resources
+			computed.Resources = resources
+		}
+		placed = append(placed, computed)
+	}
+	if len(generated) == 0 {
+		return generated, nil // nothing resolution has not already judged
+	}
+	if problems := checkResources(placed); len(problems) > 0 {
+		return nil, fmt.Errorf("what the mesh computes for this machine collides with a module's own: %s",
+			strings.Join(problems, "; "))
+	}
+	return generated, nil
+}

internal/catalogue/derived_for_consumer_test.go

@@ -0,0 +1,343 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// What a provider derives for each consumer, said once and delivered to both ends
+// (novox/hq ADR 0201, issue 124).
+//
+// The failure these are written against: the object store's provisioner derived each consumer's
+// bucket from the login the mesh minted, in its own code, and the mesh had no channel to tell the
+// consumer which bucket that was — so all three consumers wrote the answer into their own
+// definitions by hand. Two were right. One named a predecessor's bucket and would have
+// authenticated successfully and been refused on every object. Each of them also named the
+// machine the module happens to run on, which a definition may not do.
+
+// store is an object store in the shape minio has: it serves a region and a port to everyone, and
+// a bucket named for whoever is asking.
+func store() Manifest {
+	return Manifest{
+		Module: "store", Version: "1",
+		Provides: FromAnywhere("s3-bucket"),
+		Listens:  []Listening{{Port: 9000, Protocol: "tcp", From: FromMesh}},
+		Serves: map[string]map[string]any{"s3-bucket": {
+			"region": "eu-west",
+			"bucket": "${consumer:as:dns}",
+		}},
+		Receives: map[string]string{"s3-bucket": "/var/lib/store/grants/mesh.json"},
+		Grants:   map[string]string{"s3-bucket": "/var/lib/store/grants"},
+		Resources: []map[string]any{{
+			"id": "server", "type": "container", "name": "store", "ports": []any{"9000"},
+		}},
+	}
+}
+
+// files is a consumer that writes the bucket into its own configuration — which is the thing it
+// could not do before, and had to transcribe.
+func files() Manifest {
+	return Manifest{
+		Module: "files", Version: "1", Slug: "files",
+		Requires: []string{"s3-bucket"},
+		Binds:    map[string]string{"s3-bucket": "/var/lib/files/store.json"},
+		Secrets:  map[string]string{"s3-bucket": "/var/lib/files/store.secret"},
+		Resources: []map[string]any{{
+			"id": "env", "type": "file", "path": "/var/lib/files/env", "mode": "0600",
+			"content": "BUCKET=${bound:s3-bucket:bucket}\nREGION=${bound:s3-bucket:region}\n",
+		}},
+	}
+}
+
+// pics is a second consumer of the same provider on the same machine: two derivations, neither
+// the other's.
+func pics() Manifest {
+	return Manifest{
+		Module: "pics", Version: "1", Slug: "pics",
+		Requires: []string{"s3-bucket"},
+		Binds:    map[string]string{"s3-bucket": "/var/lib/pics/store.json"},
+		Secrets:  map[string]string{"s3-bucket": "/var/lib/pics/store.secret"},
+		Resources: []map[string]any{{
+			"id": "env", "type": "file", "path": "/var/lib/pics/env", "mode": "0600",
+			"content": "BUCKET=${bound:s3-bucket:bucket}\n",
+		}},
+	}
+}
+
+// The three places the derived value lands must agree, because agreeing is the whole point: the
+// consumer's own file, the binding it reads as JSON, and the provider's contributions entry.
+func TestADerivedValueReachesBothEndsAndAgrees(t *testing.T) {
+	r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := r.Declaration(Rendering{Grants: []Grant{{
+		Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+		Values: map[string]any{}, Sealed: "c2VhbGVk",
+	}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The mesh minted this identity for the consumer; the bucket is that identity as a DNS label.
+	// Derived here with the mesh's own function, so the test cannot agree with a wrong rule.
+	as := ConsumerIdentity("workstation", IdentitySource("files", "files"))
+	want := strings.ReplaceAll(as, "_", "-")
+	if want == as || !strings.Contains(as, "_") {
+		t.Fatalf("the mesh's identity %q has no separator to rewrite; this test proves nothing", as)
+	}
+
+	env := fileNamed(out, "files.env")
+	if env == nil {
+		t.Fatalf("the consumer was given no file: %v", out)
+	}
+	if got := env["content"].(string); !strings.Contains(got, "BUCKET="+want+"\n") {
+		t.Errorf("the consumer's own file was not told the bucket:\n%s\nwant BUCKET=%s", got, want)
+	}
+
+	binding := fileNamed(out, "files.bound-s3-bucket")
+	if binding == nil {
+		t.Fatalf("the consumer was given no binding: %v", out)
+	}
+	var said struct {
+		Serves map[string]any `json:"serves"`
+	}
+	if err := json.Unmarshal([]byte(binding["content"].(string)), &said); err != nil {
+		t.Fatal(err)
+	}
+	if said.Serves["bucket"] != want {
+		t.Errorf("the binding says the bucket is %q, want %q", said.Serves["bucket"], want)
+	}
+	// And what is the same for everybody is still the same for everybody.
+	if said.Serves["region"] != "eu-west" {
+		t.Errorf("the binding lost what the provider serves to all: %v", said.Serves)
+	}
+
+	given := storeGrants(t, out)
+	if len(given) != 1 {
+		t.Fatalf("the provider was told about %d consumer(s): %v", len(given), given)
+	}
+	if given[0].Derived["bucket"] != want {
+		t.Errorf("the provider was told the bucket is %v, and the consumer was told %q — "+
+			"the two ends disagree, which is the whole failure", given[0].Derived["bucket"], want)
+	}
+	// Only the per-consumer half. The region is the same for everyone and is already in the
+	// provider's own definition; repeating it here would be a copy to go stale.
+	if _, carried := given[0].Derived["region"]; carried {
+		t.Errorf("the provider was handed back what it already says for everyone: %v", given[0].Derived)
+	}
+}
+
+// Two consumers of one provider on one machine get two buckets, and neither gets the other's.
+func TestTwoConsumersOfOneProviderGetTheirOwnDerivation(t *testing.T) {
+	r, err := Resolve(shelf(store(), files(), pics()),
+		[]string{"store", "files", "pics"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := r.Declaration(Rendering{Grants: []Grant{
+		{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+			Values: map[string]any{}, Sealed: "c2VhbGVk"},
+		{Provision: "s3-bucket", Consumer: "workstation", From: "pics", Slug: "pics",
+			Values: map[string]any{}, Sealed: "c2VhbGVk"},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	forFiles := strings.ReplaceAll(ConsumerIdentity("workstation", IdentitySource("files", "files")), "_", "-")
+	forPics := strings.ReplaceAll(ConsumerIdentity("workstation", IdentitySource("pics", "pics")), "_", "-")
+	if forFiles == forPics {
+		t.Fatal("the two consumers were given the same identity; this test proves nothing")
+	}
+	if got := fileNamed(out, "files.env")["content"].(string); !strings.Contains(got, "BUCKET="+forFiles+"\n") {
+		t.Errorf("files was not given its own bucket:\n%s", got)
+	}
+	if got := fileNamed(out, "pics.env")["content"].(string); !strings.Contains(got, "BUCKET="+forPics+"\n") {
+		t.Errorf("pics was not given its own bucket:\n%s", got)
+	}
+	var buckets []any
+	for _, g := range storeGrants(t, out) {
+		buckets = append(buckets, g.Derived["bucket"])
+	}
+	if len(buckets) != 2 || buckets[0] == buckets[1] {
+		t.Errorf("the provider was told %v; it must be told one bucket per consumer", buckets)
+	}
+}
+
+// An operator may still set what the provider serves, and the mesh still derives the rest: the
+// setting is laid on first, then the consumer's half is filled.
+func TestASettingComposesWithADerivedValue(t *testing.T) {
+	r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := r.Declaration(Rendering{
+		Settings: SettingsBy{"store": {{From: "the operator",
+			Values: map[string]any{"bucket": "team-${consumer:as:dns}"}}}},
+		Grants: []Grant{{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+			Values: map[string]any{}, Sealed: "c2VhbGVk"}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "team-" + strings.ReplaceAll(ConsumerIdentity("workstation", IdentitySource("files", "files")), "_", "-")
+	if got := fileNamed(out, "files.env")["content"].(string); !strings.Contains(got, "BUCKET="+want+"\n") {
+		t.Errorf("the operator's prefix did not survive the derivation:\n%s\nwant BUCKET=%s", got, want)
+	}
+	if given := storeGrants(t, out); given[0].Derived["bucket"] != want {
+		t.Errorf("the provider was told %v, the consumer %q", given[0].Derived["bucket"], want)
+	}
+}
+
+// A fact or an alphabet the mesh does not have is refused where the definition is, not where a
+// consumer happens to be resolved — and the refusal says what may be said instead.
+func TestAServedValueNamingSomethingTheMeshDoesNotHaveIsRefused(t *testing.T) {
+	for _, c := range []struct{ value, says string }{
+		{"${consumer:node}", "as"},
+		{"${consumer:as:punycode}", "dns"},
+	} {
+		m := store()
+		m.Serves["s3-bucket"]["bucket"] = c.value
+		raw, err := json.Marshal(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = ParseManifest(raw)
+		if err == nil {
+			t.Fatalf("%s was accepted", c.value)
+		}
+		if !strings.Contains(err.Error(), c.value) {
+			t.Errorf("the refusal of %s does not quote it: %v", c.value, err)
+		}
+		if !strings.Contains(err.Error(), c.says) {
+			t.Errorf("the refusal of %s does not say what may be said (%q): %v", c.value, c.says, err)
+		}
+	}
+}
+
+// `dns` is checked against an identity the mesh actually mints, not an invented string.
+func TestTheDNSAlphabetIsTheMintedIdentityWithItsSeparatorRewritten(t *testing.T) {
+	as := ConsumerIdentity("anchor", IdentitySource("ncloud", "nextcloud"))
+	if err := CheckIdentity("anchor", IdentitySource("ncloud", "nextcloud")); err != nil {
+		t.Fatalf("the mesh would not mint this identity at all: %v", err)
+	}
+	label := asDNSLabel(as)
+	if strings.Contains(label, "_") {
+		t.Errorf("%q is not a DNS label", label)
+	}
+	if strings.ReplaceAll(label, "-", "_") != as {
+		t.Errorf("%q is not %q with its separator rewritten", label, as)
+	}
+}
+
+// The check that would have caught the one wrong instance: a consumer that writes the derived
+// value into its own definition instead of asking for it is refused, whether it transcribed the
+// right answer or a predecessor's.
+func TestAConsumerThatTranscribesWhatItsProviderDerivesIsRefused(t *testing.T) {
+	as := ConsumerIdentity("workstation", IdentitySource("files", "files"))
+	transcribed := strings.ReplaceAll(as, "_", "-")
+
+	m := files()
+	m.Resources = []map[string]any{{
+		"id": "env", "type": "file", "path": "/var/lib/files/env", "mode": "0600",
+		// Exactly what the provider will create — correct today, and a copy of a rule that is
+		// not this module's.
+		"content": "BUCKET=" + transcribed + "\n",
+	}}
+	r, err := Resolve(shelf(store(), m), []string{"store", "files"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = r.Declaration(Rendering{Grants: []Grant{{
+		Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+		Values: map[string]any{}, Sealed: "c2VhbGVk",
+	}}})
+	if err == nil {
+		t.Fatal("a definition holding its own copy of the provider's naming rule was accepted")
+	}
+	if !strings.Contains(err.Error(), "${bound:s3-bucket:bucket}") {
+		t.Errorf("the refusal does not say what to write instead: %v", err)
+	}
+
+	// And a constant the provider serves to everyone is not a transcription: repeating it is
+	// redundant, not wrong, and refusing it would be the mesh policing style.
+	m.Resources = []map[string]any{{
+		"id": "env", "type": "file", "path": "/var/lib/files/env", "mode": "0600",
+		"content": "REGION=eu-west\n",
+	}}
+	r, err = Resolve(shelf(store(), m), []string{"store", "files"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := r.Declaration(Rendering{Grants: []Grant{{
+		Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+		Values: map[string]any{}, Sealed: "c2VhbGVk",
+	}}}); err != nil {
+		t.Errorf("a value the provider serves to everyone was judged a transcription: %v", err)
+	}
+}
+
+func storeGrants(t *testing.T, out []map[string]any) []Contribution {
+	t.Helper()
+	for _, r := range out {
+		if r["path"] != "/var/lib/store/grants/mesh.json" {
+			continue
+		}
+		var parsed struct {
+			Given []Contribution `json:"given"`
+		}
+		if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil {
+			t.Fatal(err)
+		}
+		return parsed.Given
+	}
+	t.Fatalf("the provider was given no contributions file: %v", out)
+	return nil
+}
+
+// A consumer that keeps SEVERAL holders of one provision is refused, rather than told one thing
+// while its provider is told another.
+//
+// **This is issue 124's own failure, one case to the side.** The mesh gives each holder its own
+// login — `mesh_node_mod_<local>` (ADR 0094) — and the provider derives from the login, so it
+// would make one resource per holder. The consumer's side has no such dimension: there is one
+// binding file per provision and one `${bound:<provision>:<key>}`, both derived from the
+// un-suffixed identity. So the provider would create `…-mod-cold` and the consumer would be
+// configured against `…-mod`: it would authenticate successfully and be refused on every object,
+// which is exactly the fault this whole record exists to end.
+//
+// Refused, loudly, at the one place that can see both halves. Lifting it means giving the
+// consumer's side a local dimension, which is a decision and not an omission.
+func TestAConsumerWithSeveralHoldersOfADerivingProviderIsRefused(t *testing.T) {
+	m := files()
+	// Two holders of the one provision, the shape ADR 0094 gives a module that keeps several.
+	m.Secrets = nil
+	m.SecretsMany = map[string]map[string]string{"s3-bucket": {
+		"hot":  "/var/lib/files/hot.secret",
+		"cold": "/var/lib/files/cold.secret",
+	}}
+	m.Resources = []map[string]any{{
+		"id": "env", "type": "file", "path": "/var/lib/files/env", "mode": "0600",
+		"content": "BUCKET=${bound:s3-bucket:bucket}\n",
+	}}
+	r, err := Resolve(shelf(store(), m), []string{"store", "files"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = r.Declaration(Rendering{Grants: []Grant{
+		{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+			Local: "hot", Values: map[string]any{}, Sealed: "c2VhbGVk"},
+		{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
+			Local: "cold", Values: map[string]any{}, Sealed: "c2VhbGVk"},
+	}})
+	if err == nil {
+		t.Fatal("a consumer with several holders of a deriving provider was accepted; " +
+			"its two ends would have disagreed in silence")
+	}
+	for _, want := range []string{"files", "s3-bucket", "bucket"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not name %q: %v", want, err)
+		}
+	}
+}

internal/catalogue/environment_into.go

@@ -0,0 +1,593 @@
+package catalogue
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// The account's environment and the login shell's code, composed from the modules a node runs
+// (novox/hq ADR 0203, ADR 0204).
+//
+// **The same shape as the jails.** Every module may contribute — a toolchain its directory on PATH,
+// a version manager a variable naming its home, a prompt the code that loads it — naming no node, no
+// path and no file of the shell's (ADR 0112). The one module holding the matching seat places the
+// result with a placeholder in its own file, and the controller fills it from every module on the
+// node. A node not running a module has none of its contribution, and unassigning one takes its
+// lines away at the next composition.
+//
+// **Two kinds of contribution, kept apart on purpose.** The environment is facts, which the
+// controller writes in two standard formats — POSIX assignment and the service manager's
+// environment.d — so a terminal, a script, the login shell's `execute` and a graphical session all
+// read the same values (ADR 0203). Shell code is not a fact: it is text in one shell's syntax, which
+// the controller sorts into a slot and pastes without reading, as it pastes a jail's stanza (ADR
+// 0204).
+
+// EnvironmentSeat and LoginShellSeat are the seats whose holders may place what the modules
+// contributed: the account's environment, and the login shell's code.
+const (
+	EnvironmentSeat = "node-environment"
+	LoginShellSeat  = "node-login-shell"
+	// PowerSeat is the seat whose holder places code for the power moments (novox/hq ADR 0211).
+	PowerSeat = "node-power"
+)
+
+// Where an environment entry on PATH goes: before the account's existing PATH, or after it.
+const (
+	PathAtStart = "start"
+	PathAtEnd   = "end"
+)
+
+// Environment is what one module adds to the account's environment (novox/hq ADR 0203).
+type Environment struct {
+	// Variables are names and literal values. A value may name the machine's own facts with
+	// ${machine:…}, resolved before anything is written, and nothing else that expands.
+	Variables map[string]string `json:"variables,omitempty"`
+	// Path is entries on the account's PATH, each at its start or its end, in the order declared.
+	Path []PathEntry `json:"path,omitempty"`
+}
+
+// PathEntry is one directory a module puts on the account's PATH.
+type PathEntry struct {
+	Entry string `json:"entry"`
+	At    string `json:"at"`
+}
+
+// ShellCode is one piece of code a module adds to a shell's startup (novox/hq ADR 0204).
+type ShellCode struct {
+	// For is the shell the code is written in.
+	For string `json:"for"`
+	// Slot is where it runs among the other modules' code: first, normal or last. Named rather
+	// than numbered, because every contributor would guess a number and a collision says nothing.
+	Slot string `json:"slot"`
+	// Code is never interpreted — it is the shell's syntax, and only the shell reads it.
+	Code string `json:"code"`
+}
+
+// The shells and slots a contribution may name (novox/hq ADR 0204). Closed, so a typo is a refusal
+// at the check rather than code that silently lands in no placeholder.
+var (
+	knownShells = []string{"zsh", "bash", "fish"}
+	knownSlots  = []string{"first", "normal", "last"}
+	// sessionFiles are the two files of the graphical session's start that read no directory, so a
+	// contribution to them is a slot rather than a drop-in (novox/hq ADR 0208 §4): `xinitrc` is POSIX
+	// code the session's start runs, `xresources` X resources merged at its start. Placed by the
+	// display server's holder, as a shell's slots are placed by the login shell's.
+	sessionFiles = []string{"xinitrc", "xresources"}
+	// powerMoments are the moments of a machine's power a module may run code at (novox/hq ADR
+	// 0211 §3): POSIX code run as root by node-power's holder, in module order, each piece bounded.
+	powerMoments = []string{"after-boot", "before-sleep", "after-wake", "before-shutdown", "on-mains", "on-battery"}
+)
+
+// contributionTargets is every name a contribution's `for` may take.
+func contributionTargets() []string {
+	out := append(append([]string(nil), knownShells...), sessionFiles...)
+	return append(out, powerMoments...)
+}
+
+// placerOf is the seat whose holder places a contribution for this target (novox/hq ADR 0204,
+// ADR 0208 §4).
+func placerOf(target string) string {
+	if oneOf(sessionFiles, target) {
+		return DisplayServerSeat
+	}
+	if oneOf(powerMoments, target) {
+		return PowerSeat
+	}
+	return LoginShellSeat
+}
+
+// The two renderings of the environment a holder may place (novox/hq ADR 0203, decision 3).
+const (
+	EnvironmentPOSIX   = "posix"
+	EnvironmentSystemd = "systemd"
+)
+
+// ofEnvironment and ofShell are where a holder places what was contributed: ${environment:posix},
+// ${environment:systemd} and ${shell:<shell>:<slot>}. Loose inside the braces on purpose, so a
+// misspelt key is found and refused rather than left in a file as a literal nobody reads.
+var (
+	ofEnvironment = regexp.MustCompile(`\$\{environment:([^}]*)\}`)
+	ofShell       = regexp.MustCompile(`\$\{shell:([^}]*)\}`)
+	// ofContributed is either kind of contributed text, matched together so both fill in one pass.
+	ofContributed = regexp.MustCompile(`\$\{(shell|contribution):([^}]*)\}`)
+)
+
+// variableName is a POSIX shell variable name, which is also what environment.d accepts.
+var variableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
+
+// environmentProblems is what is wrong with this module's environment contribution, from the
+// manifest alone.
+func (m Manifest) environmentProblems() []string {
+	if m.Environment == nil {
+		return nil
+	}
+	var problems []string
+	for _, n := range sortedKeys(m.Environment.Variables) {
+		switch {
+		case !variableName.MatchString(n):
+			problems = append(problems, fmt.Sprintf(
+				"%s sets the variable %q, which is not a name a shell accepts: a letter or an "+
+					"underscore, then letters, digits and underscores", m.Module, n))
+			continue
+		case n == "PATH":
+			// PATH is the one variable every module shares, so no module may set it whole: a second
+			// setter would replace the first's entries, and the account's own PATH with them.
+			problems = append(problems, fmt.Sprintf(
+				"%s sets PATH as a variable; a module adds an entry under environment.path, at the "+
+					"start or the end, and PATH is composed from every module's (novox/hq ADR 0203)", m.Module))
+			continue
+		}
+		if why := literalProblem(m.Environment.Variables[n]); why != "" {
+			problems = append(problems, fmt.Sprintf(
+				"%s sets %s to %q, which %s — %s", m.Module, n, m.Environment.Variables[n], why, literalRule))
+		}
+	}
+	seen := map[string]bool{}
+	for i, p := range m.Environment.Path {
+		switch {
+		case p.Entry == "":
+			problems = append(problems, fmt.Sprintf("%s's PATH entry %d names no directory", m.Module, i+1))
+		case strings.Contains(ofMachine.ReplaceAllString(p.Entry, ""), ":"):
+			// A colon is PATH's own separator, so an entry holding one is two entries, and the
+			// check that it is already present would look for the wrong thing.
+			problems = append(problems, fmt.Sprintf(
+				"%s puts %q on PATH, which holds a colon, PATH's own separator", m.Module, p.Entry))
+		case seen[p.Entry]:
+			problems = append(problems, fmt.Sprintf("%s puts %q on PATH twice", m.Module, p.Entry))
+		default:
+			if why := literalProblem(p.Entry); why != "" {
+				problems = append(problems, fmt.Sprintf(
+					"%s puts %q on PATH, which %s — %s", m.Module, p.Entry, why, literalRule))
+			}
+		}
+		seen[p.Entry] = true
+		if p.At != PathAtStart && p.At != PathAtEnd {
+			problems = append(problems, fmt.Sprintf(
+				"%s puts %q on PATH at %q; an entry goes at %q or %q of the account's PATH",
+				m.Module, p.Entry, p.At, PathAtStart, PathAtEnd))
+		}
+	}
+	return problems
+}
+
+// literalRule is why a value must be literal, said with every refusal of one.
+const literalRule = "a value is literal, so a POSIX shell and the service manager read it alike, and " +
+	"names the machine only through the mesh's own ${machine:…} facts (novox/hq ADR 0203)"
+
+// literalProblem is why a value cannot be written, unquoted by either reader, as the same string in
+// both formats — or nothing. A `$` would expand differently in each; a quote or a backslash is
+// quoting in one and a character in the other; a line break ends the line in both.
+func literalProblem(v string) string {
+	switch {
+	case strings.ContainsAny(v, `'"`):
+		return "holds a quote"
+	case strings.Contains(v, `\`):
+		return "holds a backslash"
+	case strings.ContainsAny(v, "\n\r"):
+		return "holds a line break"
+	case strings.ContainsRune(v, 0):
+		return "holds a NUL"
+	case strings.Contains(ofMachine.ReplaceAllString(v, ""), "$"):
+		return "holds a $ that is not one of the machine's ${machine:…} facts"
+	}
+	return ""
+}
+
+// shellProblems is what is wrong with this module's shell code, from the manifest alone. The code
+// itself is not judged: it is the shell's syntax, which the controller does not read.
+func (m Manifest) shellProblems() []string {
+	var problems []string
+	for i, c := range m.Shell {
+		if !oneOf(contributionTargets(), c.For) {
+			problems = append(problems, fmt.Sprintf(
+				"%s's shell code %d is for %q; the shells are %s, the session's files %s, and the power "+
+					"moments %s", m.Module, i+1, c.For, strings.Join(knownShells, ", "),
+				strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", ")))
+		}
+		if !oneOf(knownSlots, c.Slot) {
+			problems = append(problems, fmt.Sprintf(
+				"%s's shell code %d goes in the slot %q; the slots are %s", m.Module, i+1, c.Slot,
+				strings.Join(knownSlots, ", ")))
+		}
+		if strings.TrimSpace(c.Code) == "" {
+			problems = append(problems, fmt.Sprintf("%s's shell code %d has no code", m.Module, i+1))
+		}
+	}
+	return problems
+}
+
+// contributionPlaceholderProblems is every place this module's resources name the environment or
+// the shell's code and may not — judged from the manifest, so the catalogue check refuses it before
+// a mesh does, and again at composition in the same words.
+func (m Manifest) contributionPlaceholderProblems() []string {
+	var problems []string
+	for _, r := range m.Resources {
+		problems = append(problems, placeholderProblems(m, r)...)
+		problems = append(problems, seatPlaceholderProblems(m, r)...)
+	}
+	return problems
+}
+
+// placeholderProblems is what is wrong with one resource's ${environment:…} and ${shell:…}.
+//
+// **The seat authorises it, not the placeholder** (novox/hq ADR 0203 §5, ADR 0204 §3), as the seat
+// authorises the bus's user list: a module that does not hold the account's environment writing it
+// would be a second writer of a file there is one of, and a module that does not hold the login
+// shell writing every module's shell code would be a second shell.
+func placeholderProblems(m Manifest, r map[string]any) []string {
+	var problems []string
+	for _, field := range sortedKeys(r) {
+		s, ok := r[field].(string)
+		if !ok {
+			continue
+		}
+		env := ofEnvironment.FindAllStringSubmatch(s, -1)
+		code := ofShell.FindAllStringSubmatch(s, -1)
+		if len(env)+len(code) == 0 {
+			continue
+		}
+		if field != "content" {
+			// Placed only where a file's bytes are, which is where every one of them is meant to go:
+			// a path or an owner holding several lines of shell is nothing the host could act on.
+			problems = append(problems, fmt.Sprintf(
+				"%s's resource %v names %s in its %s; the environment and the shell's code are placed "+
+					"only in a file's content", m.Module, r["id"], placeholderOf(env, code), field))
+			continue
+		}
+		for _, e := range env {
+			if e[1] != EnvironmentPOSIX && e[1] != EnvironmentSystemd {
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %v names %s; the environment is ${environment:%s} or ${environment:%s}",
+					m.Module, r["id"], e[0], EnvironmentPOSIX, EnvironmentSystemd))
+			}
+		}
+		if len(env) > 0 && !m.ClaimsSeat(EnvironmentSeat) {
+			problems = append(problems, fmt.Sprintf(
+				"%s's resource %v names %s and %s does not claim %s; the account's environment is "+
+					"written by that seat's holder alone (novox/hq ADR 0203)",
+				m.Module, r["id"], env[0][0], m.Module, EnvironmentSeat))
+		}
+		// Each placeholder judged by its own target: a shell's code is the login shell's holder's to
+		// place (ADR 0204), the session's files the display server's (ADR 0208 §4) — and a holder of
+		// one placing the other's would be a second writer of a file there is one of.
+		refusedFor := map[string]bool{}
+		for _, c := range code {
+			target, slot, two := strings.Cut(c[1], ":")
+			if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) {
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %v names %s; shell code is ${shell:<shell>:<slot>}, the shell one of "+
+						"%s, the session's file one of %s or the power moment one of %s, and the slot one of %s",
+					m.Module, r["id"], c[0], strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "),
+					strings.Join(powerMoments, ", "), strings.Join(knownSlots, ", ")))
+				continue
+			}
+			seat := placerOf(target)
+			if m.ClaimsSeat(seat) || refusedFor[seat] {
+				continue
+			}
+			refusedFor[seat] = true
+			if seat == PowerSeat {
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %v names %s and %s does not claim %s; every module's code for a power "+
+						"moment is placed by the power seat's holder alone (novox/hq ADR 0211)",
+					m.Module, r["id"], c[0], m.Module, seat))
+				continue
+			}
+			if seat == DisplayServerSeat {
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %v names %s and %s does not claim %s; every module's %s is placed by "+
+						"the display server's holder alone (novox/hq ADR 0208)",
+					m.Module, r["id"], c[0], m.Module, seat, target))
+				continue
+			}
+			problems = append(problems, fmt.Sprintf(
+				"%s's resource %v names %s and %s does not claim %s; every module's shell code is "+
+					"placed by the login shell's holder alone (novox/hq ADR 0204)",
+				m.Module, r["id"], c[0], m.Module, seat))
+		}
+	}
+	return problems
+}
+
+func placeholderOf(env, code [][]string) string {
+	if len(env) > 0 {
+		return env[0][0]
+	}
+	return code[0][0]
+}
+
+// contributedEnvironment is one node's environment, gathered and in the order it is written.
+type contributedEnvironment struct {
+	// variables is by module in name order, each module's sorted by name.
+	variables []setBy
+	// start and end are PATH's entries in their final order, each once.
+	start, end []placedOn
+}
+
+type setBy struct {
+	module string
+	names  []string
+	values map[string]string
+}
+
+type placedOn struct {
+	module, entry string
+}
+
+// inModuleOrder is the modules sorted by name — the order contributions are written in (novox/hq
+// ADR 0203, ADR 0204), so the same set composes byte for byte whatever order they were assigned in.
+func inModuleOrder(modules []Manifest) []Manifest {
+	out := append([]Manifest(nil), modules...)
+	sort.SliceStable(out, func(a, b int) bool { return out[a].Module < out[b].Module })
+	return out
+}
+
+// variablesSetOnce refuses a variable two modules on one node both set (novox/hq ADR 0203 §5),
+// naming both. Neither is chosen: whichever was written last would win in one reader and not
+// necessarily in the other, and the module that lost would not be told.
+func variablesSetOnce(modules []Manifest) error {
+	setter := map[string]string{}
+	for _, m := range inModuleOrder(modules) {
+		if m.Environment == nil {
+			continue
+		}
+		for _, n := range sortedKeys(m.Environment.Variables) {
+			if first, taken := setter[n]; taken {
+				return fmt.Errorf(
+					"%s and %s both set %s on this machine; the account has one environment, so one "+
+						"of them must stop setting it (novox/hq ADR 0203)", first, m.Module, n)
+			}
+			setter[n] = m.Module
+		}
+	}
+	return nil
+}
+
+// environmentOn gathers every module's environment on a node, with the machine's facts in place.
+//
+// A PATH entry two modules both add is written once, where the first puts it: two toolchains
+// sharing ~/.local/bin is ordinary, and nothing about it is in conflict.
+func environmentOn(modules []Manifest, facts map[string]string) (contributedEnvironment, error) {
+	var env contributedEnvironment
+	if err := variablesSetOnce(modules); err != nil {
+		return env, err
+	}
+	placed := map[string]bool{}
+	for _, m := range inModuleOrder(modules) {
+		if m.Environment == nil {
+			continue
+		}
+		if len(m.Environment.Variables) > 0 {
+			set := setBy{module: m.Module, values: map[string]string{}}
+			for _, n := range sortedKeys(m.Environment.Variables) {
+				v, err := factsIn(m.Environment.Variables[n], facts, m.Module, n)
+				if err != nil {
+					return env, err
+				}
+				set.names = append(set.names, n)
+				set.values[n] = v
+			}
+			env.variables = append(env.variables, set)
+		}
+		for _, p := range m.Environment.Path {
+			entry, err := factsIn(p.Entry, facts, m.Module, "a PATH entry")
+			if err != nil {
+				return env, err
+			}
+			if strings.Contains(entry, ":") {
+				return env, fmt.Errorf("%s puts %q on PATH on this machine, which holds a colon, PATH's own separator",
+					m.Module, entry)
+			}
+			if placed[entry] {
+				continue
+			}
+			placed[entry] = true
+			if p.At == PathAtEnd {
+				env.end = append(env.end, placedOn{m.Module, entry})
+			} else {
+				env.start = append(env.start, placedOn{m.Module, entry})
+			}
+		}
+	}
+	return env, nil
+}
+
+// factsIn resolves a contributed value's ${machine:…} facts with this machine's — first, before
+// either format is written, so both say the same thing (novox/hq ADR 0203).
+func factsIn(v string, facts map[string]string, module, what string) (string, error) {
+	for _, key := range machineUsed(v) {
+		value, has := facts[key]
+		if !has {
+			return "", fmt.Errorf("%s sets %s to a value that says ${machine:%s}, and this machine says %s",
+				module, what, key, orNothing(namesOfFacts(facts)))
+		}
+		v = strings.ReplaceAll(v, fmt.Sprintf("${machine:%s}", key), value)
+	}
+	// Judged again once filled: a fact is the mesh's, and still has to be a literal both readers
+	// take alike.
+	if why := literalProblem(v); why != "" {
+		return "", fmt.Errorf("%s sets %s to %q on this machine, which %s — %s", module, what, v, why, literalRule)
+	}
+	return v, nil
+}
+
+// posix is the environment as lines a POSIX shell sources (novox/hq ADR 0203 §3): every variable
+// exported, every PATH entry added only when it is missing, so sourcing the file twice — a login
+// shell that starts another — changes nothing. POSIX sh only, because sh, bash and zsh all read it.
+//
+// The start entries are written last-first: each is put in front of PATH, so the last written ends
+// up first, and the result reads in module order, then the order each module declared.
+func (e contributedEnvironment) posix() string {
+	var b strings.Builder
+	for _, set := range e.variables {
+		fmt.Fprintf(&b, "# %s\n", set.module)
+		for _, n := range set.names {
+			fmt.Fprintf(&b, "export %s='%s'\n", n, set.values[n])
+		}
+	}
+	named := ""
+	for i := len(e.start) - 1; i >= 0; i-- {
+		p := e.start[i]
+		if p.module != named {
+			fmt.Fprintf(&b, "# %s\n", p.module)
+			named = p.module
+		}
+		fmt.Fprintf(&b, "case \":${PATH}:\" in *':%s:'*) ;; *) PATH='%s'\"${PATH:+:${PATH}}\" ;; esac\n",
+			p.entry, p.entry)
+	}
+	named = ""
+	for _, p := range e.end {
+		if p.module != named {
+			fmt.Fprintf(&b, "# %s\n", p.module)
+			named = p.module
+		}
+		fmt.Fprintf(&b, "case \":${PATH}:\" in *':%s:'*) ;; *) PATH=\"${PATH:+${PATH}:}\"'%s' ;; esac\n",
+			p.entry, p.entry)
+	}
+	if len(e.start)+len(e.end) > 0 {
+		b.WriteString("export PATH\n")
+	}
+	return b.String()
+}
+
+// systemd is the same environment as the service manager's environment.d reads it (novox/hq ADR
+// 0203 §3), for the account's user manager and so for everything a graphical session starts. Read
+// once per manager start, so it needs no guard against running twice; the account's existing PATH
+// sits between the start and the end entries.
+func (e contributedEnvironment) systemd() string {
+	var b strings.Builder
+	for _, set := range e.variables {
+		fmt.Fprintf(&b, "# %s\n", set.module)
+		for _, n := range set.names {
+			fmt.Fprintf(&b, "%s=%s\n", n, set.values[n])
+		}
+	}
+	if len(e.start) > 0 {
+		fmt.Fprintf(&b, "# %s\nPATH=%s${PATH:+:$PATH}\n", modulesOf(e.start), entriesOf(e.start))
+	}
+	if len(e.end) > 0 {
+		fmt.Fprintf(&b, "# %s\nPATH=${PATH:+$PATH:}%s\n", modulesOf(e.end), entriesOf(e.end))
+	}
+	return b.String()
+}
+
+// modulesOf names who contributed a line holding several modules' entries, in the order they appear.
+func modulesOf(entries []placedOn) string {
+	var names []string
+	seen := map[string]bool{}
+	for _, p := range entries {
+		if !seen[p.module] {
+			seen[p.module] = true
+			names = append(names, p.module)
+		}
+	}
+	return strings.Join(names, ", ")
+}
+
+func entriesOf(entries []placedOn) string {
+	out := make([]string, len(entries))
+	for i, p := range entries {
+		out[i] = p.entry
+	}
+	return strings.Join(out, ":")
+}
+
+// shellCode is every module's code for one shell and one slot (novox/hq ADR 0204 §3): in module
+// order, each module's pieces in the order it declared them, each preceded by a line naming the
+// module, and empty when nothing is contributed.
+func shellCode(modules []Manifest, shell, slot string) string {
+	var b strings.Builder
+	for _, m := range inModuleOrder(modules) {
+		named := false
+		for _, c := range m.Shell {
+			if c.For != shell || c.Slot != slot {
+				continue
+			}
+			if !named {
+				fmt.Fprintf(&b, "# %s\n", m.Module)
+				named = true
+			}
+			b.WriteString(c.Code)
+			if !strings.HasSuffix(c.Code, "\n") {
+				b.WriteString("\n")
+			}
+		}
+	}
+	return b.String()
+}
+
+// contributionsInto fills a holder's file with the node's environment and its shell code.
+//
+// **Last, after every other placeholder pass, and in one pass each.** Shell code is contributed text
+// in a shell's own syntax — `${XDG_CACHE_HOME:-$HOME/.cache}`, `${(%):-%n}` — and the rendered
+// environment holds `${PATH:+…}`: a scanner for the mesh's own placeholders that ran after these
+// were in place would read the shell's expansions as the mesh's and refuse them, or fill a
+// `${machine:…}` some module wrote for its shell to see. So nothing runs after them, the environment
+// is filled before the shell's code is, and each is replaced in a single pass over what the holder
+// wrote, so a contributed piece is never scanned again.
+func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, facts map[string]string, with Rendering) error {
+	if problems := append(placeholderProblems(m, resource), seatPlaceholderProblems(m, resource)...); len(problems) > 0 {
+		return fmt.Errorf("%s", problems[0])
+	}
+	content, ok := resource["content"].(string)
+	if !ok {
+		return nil
+	}
+	if ofEnvironment.MatchString(content) {
+		env, err := environmentOn(modules, facts)
+		if err != nil {
+			return err
+		}
+		content = ofEnvironment.ReplaceAllStringFunc(content, func(placeholder string) string {
+			if ofEnvironment.FindStringSubmatch(placeholder)[1] == EnvironmentSystemd {
+				return env.systemd()
+			}
+			return env.posix()
+		})
+	}
+	// Shell code and seat contributions in one pass (novox/hq ADR 0212): both are contributed text
+	// the controller does not read, so neither may be scanned after the other is in place — a
+	// contributed line that happened to spell the other's placeholder would be filled.
+	if ofContributed.MatchString(content) {
+		var failed error
+		content = ofContributed.ReplaceAllStringFunc(content, func(placeholder string) string {
+			found := ofContributed.FindStringSubmatch(placeholder)
+			first, second, _ := strings.Cut(found[2], ":")
+			if found[1] == "contribution" {
+				placed, err := seatContributions(modules, first, second, with)
+				if err != nil && failed == nil {
+					failed = err
+				}
+				return placed
+			}
+			return shellCode(modules, first, second)
+		})
+		if failed != nil {
+			return failed
+		}
+	}
+	resource["content"] = content
+	return nil
+}

internal/catalogue/environment_into_test.go

@@ -0,0 +1,471 @@
+package catalogue
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Defends novox/hq ADR 0203 (the account's environment is one module's, and every module
+// contributes to it) and ADR 0204 (shell code in named slots, placed by the login shell's holder).
+
+// contributors is a fixed set of contributions, in no particular order: what the renderings are
+// asserted against byte for byte. go-toolchain and zsh both put ~/.local/bin on PATH, which is the
+// ordinary case of two modules sharing a directory, and is written once.
+func contributors() []Manifest {
+	return []Manifest{
+		{Module: "zsh", Environment: &Environment{
+			Variables: map[string]string{"XDG_CONFIG_HOME": "${machine:account-home}/.config", "EDITOR": "vim"},
+			Path: []PathEntry{
+				{Entry: "${machine:account-home}/.local/bin", At: PathAtStart},
+				{Entry: "${machine:account-home}/bin", At: PathAtStart},
+				{Entry: "/opt/scripts", At: PathAtEnd},
+			},
+		}},
+		{Module: "go-toolchain", Environment: &Environment{
+			Variables: map[string]string{"GOPATH": "${machine:account-home}/go"},
+			Path: []PathEntry{
+				{Entry: "${machine:account-home}/go/bin", At: PathAtStart},
+				{Entry: "/usr/local/go/bin", At: PathAtStart},
+				{Entry: "${machine:account-home}/.local/bin", At: PathAtStart},
+			},
+		}},
+		{Module: "agent", Environment: &Environment{
+			Variables: map[string]string{"DISABLE_AUTOUPDATER": "1"},
+			Path:      []PathEntry{{Entry: "/opt/agent/bin", At: PathAtEnd}},
+		}},
+		// A module contributing nothing is in the set and writes nothing.
+		{Module: "postgres"},
+	}
+}
+
+var operatorFacts = map[string]string{"name": "workstation", "account": "op", "account-home": "/home/op"}
+
+// The final PATH this set composes, around whatever the account had: the start entries in module
+// order and then declared order, the account's own, then the end entries.
+const composedPOSIX = `# agent
+export DISABLE_AUTOUPDATER='1'
+# go-toolchain
+export GOPATH='/home/op/go'
+# zsh
+export EDITOR='vim'
+export XDG_CONFIG_HOME='/home/op/.config'
+# zsh
+case ":${PATH}:" in *':/home/op/bin:'*) ;; *) PATH='/home/op/bin'"${PATH:+:${PATH}}" ;; esac
+# go-toolchain
+case ":${PATH}:" in *':/home/op/.local/bin:'*) ;; *) PATH='/home/op/.local/bin'"${PATH:+:${PATH}}" ;; esac
+case ":${PATH}:" in *':/usr/local/go/bin:'*) ;; *) PATH='/usr/local/go/bin'"${PATH:+:${PATH}}" ;; esac
+case ":${PATH}:" in *':/home/op/go/bin:'*) ;; *) PATH='/home/op/go/bin'"${PATH:+:${PATH}}" ;; esac
+# agent
+case ":${PATH}:" in *':/opt/agent/bin:'*) ;; *) PATH="${PATH:+${PATH}:}"'/opt/agent/bin' ;; esac
+# zsh
+case ":${PATH}:" in *':/opt/scripts:'*) ;; *) PATH="${PATH:+${PATH}:}"'/opt/scripts' ;; esac
+export PATH
+`
+
+const composedSystemd = `# agent
+DISABLE_AUTOUPDATER=1
+# go-toolchain
+GOPATH=/home/op/go
+# zsh
+EDITOR=vim
+XDG_CONFIG_HOME=/home/op/.config
+# go-toolchain, zsh
+PATH=/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin${PATH:+:$PATH}
+# agent, zsh
+PATH=${PATH:+$PATH:}/opt/agent/bin:/opt/scripts
+`
+
+func TestTheEnvironmentRendersForAPOSIXShellByteForByte(t *testing.T) {
+	env, err := environmentOn(contributors(), operatorFacts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := env.posix(); got != composedPOSIX {
+		t.Fatalf("the POSIX rendering is\n%s\nnot\n%s", got, composedPOSIX)
+	}
+}
+
+func TestTheEnvironmentRendersForTheServiceManagerByteForByte(t *testing.T) {
+	env, err := environmentOn(contributors(), operatorFacts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := env.systemd(); got != composedSystemd {
+		t.Fatalf("the environment.d rendering is\n%s\nnot\n%s", got, composedSystemd)
+	}
+}
+
+// Sourcing twice changes nothing (ADR 0203 §3): a login shell that starts another reads the file
+// again, and a PATH that grew each time would be the symptom. Run by a real `sh`, because the claim
+// is about what a shell does with the file, not about what the file looks like.
+func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) {
+	sh, err := exec.LookPath("sh")
+	if err != nil {
+		t.Skip("no sh on this machine")
+	}
+	script := "PATH=/usr/bin:/bin\n" + composedPOSIX + "once=$PATH\n" + composedPOSIX +
+		`[ "$PATH" = "$once" ] || { echo "changed: $once -> $PATH"; exit 1; }` + "\n" +
+		`echo "$PATH"; echo "$GOPATH"`
+	out, err := exec.Command(sh, "-c", script).CombinedOutput()
+	if err != nil {
+		t.Fatalf("sourcing twice: %v\n%s", err, out)
+	}
+	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+	want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts"
+	if lines[0] != want {
+		t.Fatalf("PATH is %s, not %s", lines[0], want)
+	}
+	if lines[1] != "/home/op/go" {
+		t.Fatalf("GOPATH was not exported: %q", lines[1])
+	}
+	// And an entry the account already has stays where it is, and once.
+	out, err = exec.Command(sh, "-c", "PATH=/opt/scripts:/usr/bin\n"+composedPOSIX+`echo "$PATH"`).CombinedOutput()
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if got := strings.TrimSpace(string(out)); got !=
+		"/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/opt/scripts:/usr/bin:/opt/agent/bin" {
+		t.Fatalf("an entry already on PATH was added again or moved: %s", got)
+	}
+}
+
+// The environment.d rendering, read by the service manager's own generator where this machine has
+// one — the same reader an account's user manager runs, so the PATH it composes is the one asserted.
+func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) {
+	generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator"
+	if _, err := os.Stat(generator); err != nil {
+		t.Skip("no environment.d generator on this machine")
+	}
+	config := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(composedSystemd), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cmd := exec.Command(generator)
+	cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=" + config, "XDG_CONFIG_HOME=" + config}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	want := "PATH=/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts"
+	if !strings.Contains(string(out), want+"\n") || !strings.Contains(string(out), "GOPATH=/home/op/go\n") {
+		t.Fatalf("the service manager read\n%s", out)
+	}
+}
+
+// Nothing contributed renders nothing, in both formats — not an empty `export PATH`.
+func TestNoContributionsRenderNothing(t *testing.T) {
+	env, err := environmentOn([]Manifest{{Module: "postgres"}}, operatorFacts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if env.posix() != "" || env.systemd() != "" {
+		t.Fatalf("an empty environment rendered %q and %q", env.posix(), env.systemd())
+	}
+}
+
+// A ${machine:…} fact the machine does not have is refused naming the module, as a file's is.
+func TestAContributedFactTheMachineLacksIsRefused(t *testing.T) {
+	_, err := environmentOn(contributors(), map[string]string{"name": "server"})
+	if err == nil || !strings.Contains(err.Error(), "go-toolchain sets GOPATH") ||
+		!strings.Contains(err.Error(), "${machine:account-home}") {
+		t.Fatalf("a missing account home was not refused by name: %v", err)
+	}
+}
+
+// ADR 0203 §5: two modules setting one variable are refused, both named — neither silently wins.
+func TestAVariableTwoModulesSetIsRefusedNamingBoth(t *testing.T) {
+	modules := append(contributors(), Manifest{Module: "neovim", Environment: &Environment{
+		Variables: map[string]string{"EDITOR": "nvim"}}})
+	_, err := environmentOn(modules, operatorFacts)
+	if err == nil || err.Error() != "neovim and zsh both set EDITOR on this machine; the account has one "+
+		"environment, so one of them must stop setting it (novox/hq ADR 0203)" {
+		t.Fatalf("a variable set twice was not refused naming both: %v", err)
+	}
+	// And at composition, whether or not the node holds the environment.
+	r := Resolution{Node: "workstation", Account: "op", Modules: modules}
+	if _, err := r.Declaration(Rendering{}); err == nil || !strings.Contains(err.Error(), "neovim and zsh both set EDITOR") {
+		t.Fatalf("composition accepted a variable set twice: %v", err)
+	}
+}
+
+// shells contributes code for several shells and slots, in no order.
+func shells() []Manifest {
+	return []Manifest{
+		{Module: "zsh-syntax-highlighting", Shell: []ShellCode{
+			{For: "zsh", Slot: "last", Code: "source /usr/share/zsh/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh"},
+		}},
+		{Module: "powerlevel10k", Shell: []ShellCode{
+			{For: "zsh", Slot: "first", Code: "if [[ -r \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\" ]]; then\n" +
+				"  source \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\"\nfi\n"},
+			{For: "zsh", Slot: "normal", Code: "source ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme"},
+			{For: "zsh", Slot: "normal", Code: "[[ -f ~/.local/share/powerlevel10k/p10k.zsh ]] && source ~/.local/share/powerlevel10k/p10k.zsh"},
+		}},
+		{Module: "zsh-autosuggestions", Shell: []ShellCode{
+			{For: "zsh", Slot: "normal", Code: "source /usr/share/zsh/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh"},
+			{For: "bash", Slot: "normal", Code: "echo not for zsh"},
+		}},
+		{Module: "direnv", Shell: []ShellCode{
+			{For: "fish", Slot: "last", Code: "direnv hook fish | source"},
+			{For: "bash", Slot: "last", Code: "eval \"$(direnv hook bash)\""},
+		}},
+	}
+}
+
+// ADR 0204 §3: a slot holds that shell's code only, in module order, each module's pieces in the
+// order it declared them under a line naming it; empty when nothing is contributed.
+func TestShellCodeLandsInItsSlotInModuleOrderForItsShellOnly(t *testing.T) {
+	if got, want := shellCode(shells(), "zsh", "normal"), "# powerlevel10k\n"+
+		"source ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme\n"+
+		"[[ -f ~/.local/share/powerlevel10k/p10k.zsh ]] && source ~/.local/share/powerlevel10k/p10k.zsh\n"+
+		"# zsh-autosuggestions\n"+
+		"source /usr/share/zsh/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh\n"; got != want {
+		t.Fatalf("zsh's normal slot is\n%s\nnot\n%s", got, want)
+	}
+	if got, want := shellCode(shells(), "zsh", "last"), "# zsh-syntax-highlighting\n"+
+		"source /usr/share/zsh/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh\n"; got != want {
+		t.Fatalf("zsh's last slot is\n%s\nnot\n%s", got, want)
+	}
+	if got, want := shellCode(shells(), "bash", "last"), "# direnv\neval \"$(direnv hook bash)\"\n"; got != want {
+		t.Fatalf("bash's last slot is %q, not %q", got, want)
+	}
+	if got := shellCode(shells(), "fish", "first"); got != "" {
+		t.Fatalf("a slot nobody contributed to holds %q", got)
+	}
+}
+
+// The holder of node-login-shell, as WP3's zsh module writes its block, with its own zsh around the
+// slots — which holds `${…}` of the shell's own that no mesh pass may touch either.
+func zshHolder() Manifest {
+	return Manifest{Module: "zsh", Claims: []Claim{{Name: LoginShellSeat, Scope: ScopeNode}},
+		Resources: []map[string]any{
+			{"id": "zshrc", "type": "file", "path": "${machine:account-home}/.zshrc", "content": "" +
+				"${shell:zsh:first}" +
+				"PROMPT='%n@%m ${PWD/#$HOME/~} '\n" +
+				"${shell:zsh:normal}" +
+				"alias ll='ls -l'\n" +
+				"${shell:zsh:last}"},
+		}}
+}
+
+// The case the ordering exists for: contributed zsh code full of `${…}` reaches the file byte for
+// byte, because the shell's code is placed after every other placeholder pass and in one pass — a
+// scanner for the mesh's placeholders that ran after it would read `${XDG_CACHE_HOME:-…}` and
+// `${(%):-%n}` as the mesh's, or fill a `${machine:…}` some module wrote for its shell to see.
+func TestShellCodeReachesTheHoldersFileByteForByte(t *testing.T) {
+	modules := append(shells(), zshHolder(), Manifest{Module: "sly", Shell: []ShellCode{
+		{For: "zsh", Slot: "last", Code: "echo ${machine:account-home} ${secret:x} ${shell:zsh:first} ${environment:posix}"},
+	}})
+	r := Resolution{Node: "workstation", Account: "op", Modules: modules}
+	out, err := r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var zshrc map[string]any
+	for _, res := range out {
+		if res["id"] == "zsh.zshrc" {
+			zshrc = res
+		}
+	}
+	if zshrc == nil {
+		t.Fatalf("the holder's file was not composed: %v", out)
+	}
+	if zshrc["path"] != "/home/op/.zshrc" {
+		t.Fatalf("the holder's own placeholders were not filled first: %v", zshrc["path"])
+	}
+	want := "# powerlevel10k\n" +
+		"if [[ -r \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\" ]]; then\n" +
+		"  source \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\"\nfi\n" +
+		"PROMPT='%n@%m ${PWD/#$HOME/~} '\n" +
+		"# powerlevel10k\n" +
+		"source ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme\n" +
+		"[[ -f ~/.local/share/powerlevel10k/p10k.zsh ]] && source ~/.local/share/powerlevel10k/p10k.zsh\n" +
+		"# zsh-autosuggestions\n" +
+		"source /usr/share/zsh/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh\n" +
+		"alias ll='ls -l'\n" +
+		"# sly\n" +
+		"echo ${machine:account-home} ${secret:x} ${shell:zsh:first} ${environment:posix}\n" +
+		"# zsh-syntax-highlighting\n" +
+		"source /usr/share/zsh/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh\n"
+	if got := zshrc["content"]; got != want {
+		t.Fatalf("the holder's .zshrc is\n%s\nnot\n%s", got, want)
+	}
+}
+
+// The holder of node-environment places both renderings, and they are the same as rendered alone.
+func TestTheEnvironmentHolderPlacesBothRenderings(t *testing.T) {
+	holder := Manifest{Module: "node-env", Claims: []Claim{{Name: EnvironmentSeat, Scope: ScopeNode}},
+		Resources: []map[string]any{
+			{"id": "posix", "type": "file", "path": "${machine:account-home}/.config/mesh/environment.sh",
+				"content": "# The mesh's environment.\n${environment:posix}"},
+			{"id": "systemd", "type": "file", "path": "${machine:account-home}/.config/environment.d/50-mesh.conf",
+				"content": "${environment:systemd}"},
+		}}
+	r := Resolution{Node: "workstation", Account: "op", Modules: append(contributors(), holder)}
+	out, err := r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	by := map[string]any{}
+	for _, res := range out {
+		by[res["id"].(string)] = res["content"]
+	}
+	if by["node-env.posix"] != "# The mesh's environment.\n"+composedPOSIX {
+		t.Fatalf("the POSIX file is\n%v", by["node-env.posix"])
+	}
+	if by["node-env.systemd"] != composedSystemd {
+		t.Fatalf("the environment.d file is\n%v", by["node-env.systemd"])
+	}
+}
+
+// ADR 0203 §5 and ADR 0204 §3: a placeholder outside the seat's holder is refused — by the parser,
+// which is what the catalogue check and registration run, and again at composition, in the same words.
+func TestAPlaceholderOutsideTheHolderIsRefused(t *testing.T) {
+	for _, c := range []struct{ content, want string }{
+		{"${environment:posix}", "toolchain's resource rc names ${environment:posix} and toolchain does not claim node-environment"},
+		{"${shell:zsh:normal}", "toolchain's resource rc names ${shell:zsh:normal} and toolchain does not claim node-login-shell"},
+	} {
+		raw := `{"module":"toolchain","resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` + c.content + `"}]}`
+		if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("the catalogue check accepted %s outside its holder: %v", c.content, err)
+		}
+		m := Manifest{Module: "toolchain", Resources: []map[string]any{
+			{"id": "rc", "type": "file", "path": "/etc/rc", "content": c.content}}}
+		r := Resolution{Node: "workstation", Account: "op", Modules: []Manifest{m}}
+		if _, err := r.Declaration(Rendering{}); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("composition accepted %s outside its holder: %v", c.content, err)
+		}
+	}
+}
+
+// A key nobody renders is refused, not left in the file as a literal.
+func TestAnUnknownPlaceholderKeyIsRefused(t *testing.T) {
+	for _, c := range []struct{ content, want string }{
+		{"${environment:foo}", "names ${environment:foo}; the environment is ${environment:posix} or ${environment:systemd}"},
+		{"${shell:zsh:middle}", "names ${shell:zsh:middle}; shell code is ${shell:<shell>:<slot>}"},
+		{"${shell:tcsh:first}", "names ${shell:tcsh:first}; shell code is ${shell:<shell>:<slot>}"},
+		{"${shell:zsh}", "names ${shell:zsh}; shell code is ${shell:<shell>:<slot>}"},
+	} {
+		raw := `{"module":"holder","claims":[{"name":"node-environment","scope":"node"},{"name":"node-login-shell","scope":"node"}],` +
+			`"resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` + c.content + `"}]}`
+		if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s was accepted: %v", c.content, err)
+		}
+	}
+	// And outside a file's content, where nothing could be placed.
+	raw := `{"module":"holder","claims":[{"name":"node-environment","scope":"node"}],` +
+		`"resources":[{"id":"rc","type":"file","path":"/etc/${environment:posix}","content":"x"}]}`
+	if _, err := ParseManifest([]byte(raw)); err == nil ||
+		!strings.Contains(err.Error(), "names ${environment:posix} in its path; the environment and the shell's code are placed only in a file's content") {
+		t.Errorf("a placeholder in a path was accepted: %v", err)
+	}
+}
+
+// What ADR 0203 §2 allows a contribution to say, refused at parse when it says anything else.
+func TestAMalformedEnvironmentIsRefusedAtParse(t *testing.T) {
+	for _, c := range []struct{ environment, want string }{
+		{`{"variables":{"1X":"a"}}`, `tool sets the variable "1X", which is not a name a shell accepts`},
+		{`{"variables":{"MY-VAR":"a"}}`, `tool sets the variable "MY-VAR", which is not a name a shell accepts`},
+		{`{"variables":{"PATH":"/bin"}}`, `tool sets PATH as a variable; a module adds an entry under environment.path`},
+		{`{"variables":{"A":"$HOME/x"}}`, `tool sets A to "$HOME/x", which holds a $ that is not one of the machine's ${machine:…} facts`},
+		{`{"variables":{"A":"${HOME}/x"}}`, `tool sets A to "${HOME}/x", which holds a $`},
+		{`{"variables":{"A":"it's"}}`, `tool sets A to "it's", which holds a quote`},
+		{`{"variables":{"A":"say \"hi\""}}`, `which holds a quote`},
+		{`{"variables":{"A":"a\\b"}}`, `which holds a backslash`},
+		{`{"variables":{"A":"a\nb"}}`, `which holds a line break`},
+		{`{"variables":{"A":"a\u0000b"}}`, `which holds a NUL`},
+		{`{"path":[{"entry":"","at":"start"}]}`, `tool's PATH entry 1 names no directory`},
+		{`{"path":[{"entry":"/a:/b","at":"start"}]}`, `tool puts "/a:/b" on PATH, which holds a colon`},
+		{`{"path":[{"entry":"$HOME/bin","at":"start"}]}`, `tool puts "$HOME/bin" on PATH, which holds a $`},
+		{`{"path":[{"entry":"/a","at":"middle"}]}`, `tool puts "/a" on PATH at "middle"; an entry goes at "start" or "end"`},
+		{`{"path":[{"entry":"/a"}]}`, `tool puts "/a" on PATH at ""`},
+		{`{"path":[{"entry":"/a","at":"start"},{"entry":"/a","at":"end"}]}`, `tool puts "/a" on PATH twice`},
+		{`{"variables":{"A":"x"},"paths":[]}`, `unknown field "paths"`},
+	} {
+		_, err := ParseManifest([]byte(`{"module":"tool","environment":` + c.environment + `}`))
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s: want %q, got %v", c.environment, c.want, err)
+		}
+	}
+	// What is allowed: a literal, and the machine's own facts.
+	if _, err := ParseManifest([]byte(`{"module":"tool","environment":{` +
+		`"variables":{"GOPATH":"${machine:account-home}/go","DISABLE_X":"1","ANSWER":"a b+c=d"},` +
+		`"path":[{"entry":"${machine:account-home}/go/bin","at":"start"},{"entry":"/opt/x","at":"end"}]}}`)); err != nil {
+		t.Fatalf("a well-formed environment was refused: %v", err)
+	}
+}
+
+func TestMalformedShellCodeIsRefusedAtParse(t *testing.T) {
+	for _, c := range []struct{ shell, want string }{
+		{`[{"for":"tcsh","slot":"normal","code":"x"}]`, `tool's shell code 1 is for "tcsh"; the shells are zsh, bash, fish`},
+		{`[{"for":"zsh","slot":"middle","code":"x"}]`, `tool's shell code 1 goes in the slot "middle"; the slots are first, normal, last`},
+		{`[{"for":"zsh","slot":"last","code":"x"},{"for":"zsh","slot":"last","code":" \n"}]`, `tool's shell code 2 has no code`},
+		{`[{"for":"zsh","slot":"last","code":"x","order":1}]`, `unknown field "order"`},
+	} {
+		_, err := ParseManifest([]byte(`{"module":"tool","shell":` + c.shell + `}`))
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s: want %q, got %v", c.shell, c.want, err)
+		}
+	}
+	// The code itself is never judged: a shell's own `${…}` is not the mesh's.
+	if _, err := ParseManifest([]byte(`{"module":"tool","shell":[{"for":"zsh","slot":"first",` +
+		`"code":"source \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\""}]}`)); err != nil {
+		t.Fatalf("shell code was judged as if it were the mesh's: %v", err)
+	}
+}
+
+// ADR 0203 §1 and ADR 0204 §1: both seats are the mesh's own, held once per machine; the login
+// shell's contract is `execute`, described and with a schema an agent can call.
+func TestTheSeatTableCarriesTheEnvironmentAndTheLoginShell(t *testing.T) {
+	env, ok := SeatNamed("node-environment")
+	if !ok || env.Scope != ScopeNode || env.Decision != "novox/hq ADR 0203" ||
+		len(env.Serves)+len(env.Accepts)+len(env.Emits) != 0 || env.Delivers != "" {
+		t.Fatalf("node-environment is not a node seat with no protocol: %+v (defined %v)", env, ok)
+	}
+	shell, ok := SeatNamed("node-login-shell")
+	if !ok || shell.Scope != ScopeNode || shell.Decision != "novox/hq ADR 0204" {
+		t.Fatalf("node-login-shell is not a node seat: %+v (defined %v)", shell, ok)
+	}
+	if len(shell.Serves) != 1 || shell.Serves[0].Name != "execute" || shell.Serves[0].Description == "" {
+		t.Fatalf("the login shell serves %+v, not execute alone", shell.Serves)
+	}
+	props, _ := shell.Serves[0].Input["properties"].(map[string]any)
+	required, _ := shell.Serves[0].Input["required"].([]string)
+	if _, has := props["command"]; !has || len(required) != 1 || required[0] != "command" {
+		t.Fatalf("execute does not require a command: %v", shell.Serves[0].Input)
+	}
+	if _, has := props["timeout_seconds"]; !has {
+		t.Fatalf("execute takes no timeout: %v", props)
+	}
+}
+
+// ADR 0204 §1: the login shell is the mesh's, so no module declares it — neither under the mesh's
+// name nor under the name a module gave it before.
+func TestNoModuleMayDeclareTheLoginShell(t *testing.T) {
+	for _, n := range []string{"login-shell", "node-login-shell", "node-environment"} {
+		raw := `{"module":"zsh","seats":[{"name":"` + n + `","scope":"node","serves":["execute"]}],"tools":["execute"]}`
+		_, err := ParseManifest([]byte(raw))
+		if err == nil {
+			t.Errorf("a module declaring %q was accepted", n)
+		}
+	}
+	got := strings.Join(declaredSeatProblems(Manifest{Module: "zsh",
+		DefinesSeats: []SeatDeclaration{{Name: "login-shell", Scope: ScopeNode}}}), "; ")
+	if !strings.Contains(got, `zsh declares a seat named "login-shell"; the login shell is the mesh's own seat node-login-shell`) {
+		t.Fatalf("declaring login-shell was not refused by name: %q", got)
+	}
+	// And a shell module claiming the mesh's seat, serving execute, is what the seat is for.
+	m, err := ParseManifest([]byte(`{"module":"zsh","tools":["execute"],` +
+		`"claims":[{"name":"node-login-shell","scope":"node"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := CanHold(m, Seat{Name: LoginShellSeat, Scope: ScopeNode, Serves: loginShellVerbs()}); err != nil {
+		t.Fatalf("a shell module claiming the seat cannot hold it: %v", err)
+	}
+}

internal/catalogue/events.go

@@ -3,6 +3,7 @@ package catalogue
 import (
 	"fmt"
 	"regexp"
+	"slices"
 	"strings"
 )
 
@@ -159,3 +160,34 @@ func consumePattern(pattern string) error {
 	}
 	return nil
 }
+
+// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
+// without one success for minutes, and that consumer succeeding again or being withdrawn. The
+// controller follows them from every module and `status` names a consumer failing until it recovers.
+const (
+	ProvisionerFailing   = "provisioner.failing"
+	ProvisionerRecovered = "provisioner.recovered"
+)
+
+// ProvisionerEvents are both, in the order they are said.
+var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
+
+// EmitsAll is every event a module may publish: what it declares and, for a module that receives
+// contributions — a provider, running a provisioner over them — the provider's standing events.
+//
+// **Derived, not declared**, because they are the mesh's rule about every provider rather than
+// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
+// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
+// grant of a module's publishing reads this, never the declared list alone.
+func (m Manifest) EmitsAll() []string {
+	out := append([]string(nil), m.Emits...)
+	if len(m.Receives) == 0 {
+		return out
+	}
+	for _, e := range ProvisionerEvents {
+		if !slices.Contains(out, e) {
+			out = append(out, e)
+		}
+	}
+	return out
+}

internal/catalogue/foundation_manifests_test.go

@@ -1,6 +1,7 @@
 package catalogue
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"reflect"
@@ -170,7 +171,7 @@ func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) {
 // **And the forge's own address follows it**, composed from the manifest in the catalogue beside
 // this checkout (novox/hq 04-ISSUES/088).
 //
-// The forge is reached a third way that neither test above covers: by its own sidecar, over the
+// The forge is reached a third way that neither test above covers: by its own code, over the
 // machine's loopback, told where to go in its environment. The `2999:3000` mapping that lets the
 // forge go on binding 3000 does nothing for a caller dialling the machine — so a literal there is
 // wrong on every node whose assignment differs, and wrong for a second reason on a node given the
@@ -178,13 +179,13 @@ func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) {
 // in an `env` at all is a declaration, not a manifest.
 func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
 	forge, err := catalogueManifest(t, "gitea").Resolve([]Built{{
-		Name: "runtime", Kind: ArtifactImage,
+		Name: "code", Kind: ArtifactBundle,
-		Reference: "registry.example/gitea-runtime@sha256:" + strings.Repeat("a", 64),
+		Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest,
 	}})
 	if err != nil {
 		t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
 	}
-	r := Resolution{Node: "anchor", Modules: []Manifest{forge}, Needs: []Needed{
+	r := Resolution{Node: "anchor", Modules: []Manifest{forge, theRuntime(t)}, Needs: []Needed{
 		{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
 			Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
 		{Name: "route", For: "gitea", From: "anchor"},
@@ -194,8 +195,8 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
 
 	// The number this node was given for the forge — the one the machine it is about to run on
 	// already publishes.
-	out, err := r.Declaration(Rendering{
+	out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101",
-		Needed: map[string]map[string]string{"gitea": {"broker": "sealed-broker"}},
+		Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-broker"}},
 		Given:  map[string]map[int]int{"gitea": {3000: 2999}},
 	})
 	if err != nil {
@@ -210,14 +211,19 @@ func TestTheForgesOwnAddressFollowsThePortTheNodeGaveIt(t *testing.T) {
 	if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "2999:3000") {
 		t.Fatalf("the forge is not published on the port this node gave it: %v", server["ports"])
 	}
-	runtime := fileNamed(out, "gitea.runtime")
+	// The forge's own code runs in the node's runtime (novox/hq ADR 0198), given its words there.
+	runtime := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
 	if runtime == nil {
-		t.Fatalf("the forge's sidecar is not in the declaration: %v", out)
+		t.Fatalf("the node's runtime is not in the declaration: %v", ids(out))
 	}
-	env, _ := runtime["env"].(map[string]any)
+	env, _ := runtime["env"].(map[string]string)
-	if env["MESH_GITEA_URL"] != "http://127.0.0.1:2999" {
+	var given map[string]map[string]string
-		t.Fatalf("the forge's sidecar dials %v while the machine publishes the forge on 2999 — "+
+	if err := json.Unmarshal([]byte(env[RuntimeToolEnv]), &given); err != nil {
-			"whatever reads it dials a dead port", env["MESH_GITEA_URL"])
+		t.Fatalf("the runtime's %s is not JSON: %q", RuntimeToolEnv, env[RuntimeToolEnv])
+	}
+	if given["gitea"]["MESH_GITEA_URL"] != "http://127.0.0.1:2999" {
+		t.Fatalf("the forge's code dials %v while the machine publishes the forge on 2999 — "+
+			"whatever reads it dials a dead port", given["gitea"]["MESH_GITEA_URL"])
 	}
 }
 
@@ -229,13 +235,13 @@ func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
 	t.Helper()
 	forge := catalogueManifest(t, "gitea")
 	resolved, err := forge.Resolve([]Built{{
-		Name: "runtime", Kind: ArtifactImage,
+		Name: "code", Kind: ArtifactBundle,
-		Reference: "registry.example/gitea-runtime@sha256:" + strings.Repeat("a", 64),
+		Reference: ArtifactStoreScheme + "gitea/code/blobs/" + bundleDigest, Digest: bundleDigest,
 	}})
 	if err != nil {
 		t.Fatalf("the forge's manifest does not resolve against its own build: %v", err)
 	}
-	r := Resolution{Node: "anchor", Modules: []Manifest{resolved}, Needs: []Needed{
+	r := Resolution{Node: "anchor", Modules: []Manifest{resolved, theRuntime(t)}, Needs: []Needed{
 		{Name: "postgres-database", For: "gitea", From: "anchor", At: "127.0.0.1",
 			Serves: map[string]any{"port": float64(5432)}, Sealed: "sealed-db"},
 		{Name: "route", For: "gitea", From: "anchor"},
@@ -246,8 +252,8 @@ func declaredGiteaSsh(t *testing.T, given map[int]int) map[string]any {
 	for k, v := range given {
 		givenPorts[k] = v
 	}
-	out, err := r.Declaration(Rendering{
+	out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101",
-		Needed: map[string]map[string]string{"gitea": {"broker": "sealed-broker"}},
+		Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-broker"}},
 		Ports:  map[string]map[int]int{"gitea": givenPorts},
 		Given:  map[string]map[int]int{"gitea": given},
 	})

internal/catalogue/generated_collision_test.go

@@ -0,0 +1,84 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// novox/hq issue 190, step 5 (ADR 0222): what the mesh computes for a module is held to the rule
+// every module is — no two modules on a machine declare one path, unit, name or package. The
+// private network once declared the container runtime's file and service beside the runtime's own
+// module, and nothing refused it, because the collision check only ever saw catalogue manifests.
+
+func runtimesOwn() Manifest {
+	return Manifest{Module: "docker", Resources: []map[string]any{
+		{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", "into": "json",
+			"content": `{"live-restore": true}`},
+		{"id": "runtime", "type": "service", "unit": "docker.service", "state": "running",
+			"reload-on": []any{"daemon"}},
+	}}
+}
+
+func TestAGeneratedResourceCollidingWithAModulesIsRefused(t *testing.T) {
+	r := Resolution{Node: "workstation", Modules: []Manifest{
+		{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
+		runtimesOwn(),
+	}}
+	_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
+	if err == nil {
+		t.Fatal("a generated resource declaring the runtime's file beside the runtime's module was accepted")
+	}
+	for _, want := range []string{"mesh-network", "docker", "/etc/docker/daemon.json", "docker.service"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not name %s: %v", want, err)
+		}
+	}
+}
+
+func TestAGeneratedResourceBesideAModulesOwnIsComposed(t *testing.T) {
+	r := Resolution{Node: "workstation", Modules: []Manifest{
+		{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
+		runtimesOwn(),
+	}}
+	gen := &fake{on: map[string]bool{"workstation": true}}
+	out, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": gen}})
+	if err != nil {
+		t.Fatalf("disjoint resources were refused: %v", err)
+	}
+	if fileNamed(out, "docker.daemon") == nil {
+		t.Fatalf("the runtime's own file is missing: %v", out)
+	}
+	// And a machine not on the network yet generates nothing, which collides with nothing.
+	if _, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": &fake{}}}); err != nil {
+		t.Fatalf("a machine off the network was refused: %v", err)
+	}
+}
+
+// The catalogue's container runtime module states the mesh's registry itself (ADR 0222).
+func TestTheRuntimesModuleTrustsTheMeshsRegistry(t *testing.T) {
+	docker := catalogueManifest(t, "docker")
+	r := Resolution{Node: "workstation", Modules: []Manifest{docker}}
+	out, err := r.Declaration(Rendering{
+		SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	daemon := fileNamed(out, "docker.daemon")
+	if daemon == nil || daemon["into"] != "json" {
+		t.Fatalf("the runtime's file is not written into: %v", daemon)
+	}
+	content, _ := daemon["content"].(string)
+	if !strings.Contains(content, `"insecure-registries": ["anchor.internal:5100"]`) ||
+		!strings.Contains(content, `"live-restore": true`) {
+		t.Fatalf("the runtime's file says %q", content)
+	}
+
+	out, err = r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if content, _ := fileNamed(out, "docker.daemon")["content"].(string); strings.Contains(content, "insecure-registries") {
+		t.Fatalf("with no store on the network, the runtime is told %q", content)
+	}
+}

internal/catalogue/genesis_image_test.go

@@ -0,0 +1,29 @@
+package catalogue
+
+import (
+	"os"
+	"regexp"
+	"testing"
+)
+
+// novox/hq issue 223: genesis raises the controller as a container built from this repository's own
+// Dockerfile, with no build arguments — the manifest no longer builds an image, so nothing passes a
+// base in. The Dockerfile's own default must therefore be a Go that builds this module, pinned by
+// digest, and the replacement the manifest's process names must be the container genesis raises.
+func TestGenesisCanBuildTheControllersImageAsItStands(t *testing.T) {
+	raw, err := os.ReadFile("../../Dockerfile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !regexp.MustCompile(`(?m)^ARG GO_BASE=golang@sha256:[0-9a-f]{64}$`).Match(raw) {
+		t.Fatal("the Dockerfile's default Go base is not pinned by digest; genesis builds it with no arguments")
+	}
+	makefile, err := os.ReadFile("../../Makefile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pin := regexp.MustCompile(`golang@sha256:[0-9a-f]{64}`)
+	if string(pin.Find(raw)) != string(pin.Find(makefile)) {
+		t.Errorf("the Dockerfile and the Makefile build on different Go: %s, %s", pin.Find(raw), pin.Find(makefile))
+	}
+}

internal/catalogue/grant_secret_owner_test.go

@@ -0,0 +1,103 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A grant secret is read by whatever provisions, and that stopped being root (novox/hq issue 225).
+//
+// The mesh seals one credential per consumer beside the provider's contributions file. The
+// provider's harness reads both: the file to learn who asked, the secret to set their password.
+// While a module's own code ran in a container as root, a root-owned 0600 file was readable by
+// the thing that needed it. ADR 0198 moved that code under the node's runtime, which runs as the
+// operator's account — and the secret stayed root's.
+//
+// **The cost was silence.** The harness says `secret not readable yet`, which is true and
+// ordinary on the first pass, so four thousand refusals in three hours read as patience. No user
+// was ever created, and two consumers crash-looped against a database that had never heard of
+// them.
+//
+// The same reasoning is already written for a module's *own* secrets, three hundred lines above:
+// "a root-owned 0600 file is one that process cannot read". This is that rule reaching the other
+// kind of secret the mesh writes for a module.
+
+// aProviderWithABundle is a provider whose code is a bundle the node's runtime runs — the shape
+// every TypeScript provisioner has since ADR 0198.
+func aProviderWithABundle() Manifest {
+	return Manifest{
+		Module: "mongodb", Version: "1",
+		Provides: FromAnywhere("mongodb-database"),
+		Receives: map[string]string{"mongodb-database": "/var/lib/mongodb/grants/mesh.json"},
+		Grants:   map[string]string{"mongodb-database": "/var/lib/mongodb/grants"},
+		Bundles:  []Bundle{{Name: "code", Language: "typescript"}},
+		Resources: []map[string]any{{
+			"id": "server", "type": "container", "name": "mongodb-server",
+			"image": "mongo@sha256:" + strings.Repeat("a", 64),
+		}},
+	}
+}
+
+func TestAGrantSecretIsOwnedByTheAccountThatProvisions(t *testing.T) {
+	r, err := Resolve(shelf(aProviderWithABundle()), []string{"mongodb"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	r.Account = "operator"
+	out, err := r.Declaration(Rendering{Grants: []Grant{{
+		Provision: "mongodb-database", Consumer: "workstation", From: "photos", Slug: "photos",
+		Values: map[string]any{}, Sealed: "c2VhbGVk",
+	}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var secret map[string]any
+	for _, res := range out {
+		if res["type"] == "file" && strings.HasSuffix(fmtPath(res), ".secret") {
+			secret = res
+		}
+	}
+	if secret == nil {
+		t.Fatalf("no grant secret was composed at all: %v", out)
+	}
+	if got := secret["owner"]; got != "operator" {
+		t.Fatalf("the grant secret at %v belongs to %v; the provisioner runs as %q and a "+
+			"root-owned 0600 file is one it cannot read — which is silent, because the harness "+
+			"calls it \"not readable yet\"", fmtPath(secret), got, "operator")
+	}
+}
+
+// And a provider whose code still runs in a container keeps the owner it declares, so this
+// changes nothing for the modules the runtime has not taken.
+func TestAContainerProvidersGrantSecretKeepsItsDeclaredOwner(t *testing.T) {
+	m := aProviderWithABundle()
+	m.Bundles = nil
+	m.SecretsOwner = "65534:65534"
+	r, err := Resolve(shelf(m), []string{"mongodb"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	r.Account = "operator"
+	out, err := r.Declaration(Rendering{Grants: []Grant{{
+		Provision: "mongodb-database", Consumer: "workstation", From: "photos", Slug: "photos",
+		Values: map[string]any{}, Sealed: "c2VhbGVk",
+	}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, res := range out {
+		if res["type"] == "file" && strings.HasSuffix(fmtPath(res), ".secret") {
+			if got := res["owner"]; got != "65534:65534" {
+				t.Fatalf("a container provider's grant secret belongs to %v, not what it declares", got)
+			}
+			return
+		}
+	}
+	t.Fatal("no grant secret was composed")
+}
+
+func fmtPath(r map[string]any) string {
+	p, _ := r["path"].(string)
+	return p
+}

internal/catalogue/graphical_session.go

@@ -0,0 +1,109 @@
+package catalogue
+
+// The graphical session's seats (novox/hq ADR 0208): one module per piece of software, each piece's
+// role a node seat in the mesh's own set, so i3 and sway, xterm and foot, rofi and dmenu compete for
+// a role rather than each inventing one — and a machine running two of one role is refused at
+// assignment instead of found by two bars on one screen.
+const (
+	LoginManagerSeat     = "node-login-manager"
+	DisplayServerSeat    = "node-display-server"
+	DisplaySessionSeat   = "node-display-session"
+	TerminalEmulatorSeat = "node-terminal-emulator"
+	LauncherSeat         = "node-launcher"
+	NotifierSeat         = "node-notifier"
+	LockScreenSeat       = "node-lock-screen"
+	ClipboardSeat        = "node-clipboard"
+	BarSeat              = "node-bar"
+	CompositorSeat       = "node-compositor"
+	SecretServiceSeat    = "node-secret-service"
+)
+
+// graphicalSessionSeats are the eleven, in the order ADR 0208's table reads, each with the verbs
+// research 026/05 starts it with. Three have none yet: the bar, the compositor and the secret
+// service are roles a second holder competes for, and nothing has needed to ask them anything.
+func graphicalSessionSeats() []Seat {
+	const decided = "novox/hq ADR 0208"
+	return []Seat{
+		{Name: LoginManagerSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "sessions", Description: "The sessions the login manager offers on this machine, and which " +
+				"one the operator account starts by default.",
+				Input: schema(map[string]string{}, nil)},
+		}},
+		{Name: DisplayServerSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "displays", Description: "The monitors connected now, each with its identity, its modes and " +
+				"where it is placed; and the layout profile in force, if one matches.",
+				Input: schema(map[string]string{}, nil)},
+			// Profiles are keyed by the monitors' identities and are the operator's data (ADR 0208 §6).
+			{Name: "layout", Description: "The monitor layout profiles, keyed by the connected monitors' " +
+				"identities: list them, save the current arrangement under a name, or apply one.",
+				Input: withEnum(schema(map[string]string{
+					"action": "list, save or apply",
+					"name":   "the profile to save or apply (save and apply only)",
+				}, []string{"action"}), "action", "list", "save", "apply")},
+		}},
+		// It receives window-manager configuration lines from every other module (novox/hq ADR 0212):
+		// bindings, start-up commands, rules — placed by the session's holder, never written into
+		// its directory by the contributor.
+		{Name: DisplaySessionSeat, Scope: ScopeNode, Decision: decided,
+			Receives: []Receivable{{Kind: "config", Comment: "#"}}, Serves: []Verb{
+				{Name: "reload", Description: "Reload the session's configuration in place, keeping its windows.",
+					Input: schema(map[string]string{}, nil)},
+				{Name: "workspaces", Description: "The session's workspaces: each one's name, output, and whether " +
+					"it is visible or focused.",
+					Input: schema(map[string]string{}, nil)},
+				{Name: "windows", Description: "The session's windows: each one's title, class, workspace and " +
+					"whether it has focus; narrowed to one workspace when named.",
+					Input: schema(map[string]string{"workspace": "one workspace (optional)"}, nil)},
+			}},
+		{Name: TerminalEmulatorSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "open", Description: "Open a terminal window in the operator's session, running a command " +
+				"or the login shell, in a directory or the account's home.",
+				Input: schema(map[string]string{
+					"command":   "what to run in it (optional; the login shell when absent)",
+					"directory": "where it starts (optional; the account's home when absent)",
+				}, nil)},
+		}},
+		{Name: LauncherSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "menu", Description: "Put a menu of choices in front of the operator and answer with the " +
+				"one chosen, or nothing when the menu was dismissed — the dmenu-compatible contract.",
+				Input: map[string]any{"type": "object", "required": []string{"choices"},
+					"properties": map[string]any{
+						"choices": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
+							"description": "the lines to choose between, in order"},
+						"prompt": map[string]any{"type": "string", "description": "what the menu asks (optional)"},
+					}}},
+		}},
+		{Name: NotifierSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "send", Description: "Show the operator a notification.",
+				Input: withEnum(schema(map[string]string{
+					"title":   "the notification's summary",
+					"body":    "its text (optional)",
+					"urgency": "low, normal (the default) or critical",
+				}, []string{"title"}), "urgency", "low", "normal", "critical")},
+			{Name: "history", Description: "The notifications shown lately, newest first.",
+				Input: schema(map[string]string{"limit": "how many (optional, default 20)"}, nil)},
+		}},
+		{Name: LockScreenSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "lock", Description: "Lock the operator's session now.",
+				Input: schema(map[string]string{}, nil)},
+		}},
+		{Name: ClipboardSeat, Scope: ScopeNode, Decision: decided, Serves: []Verb{
+			{Name: "history", Description: "What the clipboard held lately, newest first.",
+				Input: schema(map[string]string{"limit": "how many (optional, default 20)"}, nil)},
+			{Name: "copy", Description: "Put text on the operator's clipboard.",
+				Input: schema(map[string]string{"text": "the text"}, []string{"text"})},
+		}},
+		{Name: BarSeat, Scope: ScopeNode, Decision: decided},
+		{Name: CompositorSeat, Scope: ScopeNode, Decision: decided},
+		{Name: SecretServiceSeat, Scope: ScopeNode, Decision: decided},
+	}
+}
+
+// withEnum narrows one string property of a schema to the values it may take, so a caller is told
+// the choices by the schema rather than by a refusal.
+func withEnum(s map[string]any, property string, values ...string) map[string]any {
+	props := s["properties"].(map[string]any)
+	p := props[property].(map[string]any)
+	p["enum"] = values
+	return s
+}

internal/catalogue/graphical_session_test.go

@@ -0,0 +1,213 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Defends novox/hq ADR 0208: the graphical session is one module per piece, on the mesh's seats.
+
+// §2: the eleven roles are the mesh's own node seats, each with the verbs it starts with.
+func TestTheGraphicalSessionsSeatsAreTheMeshsOwnWithTheirVerbs(t *testing.T) {
+	want := map[string][]string{
+		LoginManagerSeat:     {"sessions"},
+		DisplayServerSeat:    {"displays", "layout"},
+		DisplaySessionSeat:   {"reload", "workspaces", "windows"},
+		TerminalEmulatorSeat: {"open"},
+		LauncherSeat:         {"menu"},
+		NotifierSeat:         {"send", "history"},
+		LockScreenSeat:       {"lock"},
+		ClipboardSeat:        {"history", "copy"},
+		BarSeat:              nil,
+		CompositorSeat:       nil,
+		SecretServiceSeat:    nil,
+	}
+	for name, verbs := range want {
+		s, ok := SeatNamed(name)
+		if !ok {
+			t.Errorf("%s is not in the mesh's set", name)
+			continue
+		}
+		if s.Scope != ScopeNode || s.Decision != "novox/hq ADR 0208" {
+			t.Errorf("%s is %s-scoped under %q", name, s.Scope, s.Decision)
+		}
+		var got []string
+		for _, v := range s.Serves {
+			got = append(got, v.Name)
+			if v.Description == "" || v.Input["type"] != "object" {
+				t.Errorf("%s.%s has no description or no object schema", name, v.Name)
+			}
+		}
+		if !reflect.DeepEqual(got, verbs) {
+			t.Errorf("%s serves %v, want %v", name, got, verbs)
+		}
+	}
+	// The launcher's menu takes a list, and the layout verb says its actions.
+	menu, _ := SeatNamed(LauncherSeat)
+	choices := menu.Serves[0].Input["properties"].(map[string]any)["choices"].(map[string]any)
+	if choices["type"] != "array" {
+		t.Errorf("menu's choices are %v, not a list", choices["type"])
+	}
+	display, _ := SeatNamed(DisplayServerSeat)
+	action := display.Serves[1].Input["properties"].(map[string]any)["action"].(map[string]any)
+	if !reflect.DeepEqual(action["enum"], []string{"list", "save", "apply"}) {
+		t.Errorf("layout's actions are %v", action["enum"])
+	}
+	// And they survive the store's JSON, which is where the live set comes from.
+	if _, err := json.Marshal(graphicalSessionSeats()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// §2: a module may claim one of them, and may not declare it as its own.
+func TestNoModuleMayDeclareAGraphicalSessionSeat(t *testing.T) {
+	raw := `{"module":"xorg","seats":[{"name":"node-display-server","scope":"node"}]}`
+	if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "mesh's own namespace") {
+		t.Fatalf("a module declared node-display-server as its own: %v", err)
+	}
+}
+
+func displayServer(name, display string, claims ...string) Manifest {
+	m := Manifest{Module: name, Provides: []Offer{{Name: display, Reach: ReachMachine}}}
+	for _, c := range claims {
+		m.Claims = append(m.Claims, Claim{Name: c})
+	}
+	return m
+}
+
+func windowManager() Manifest {
+	return Manifest{Module: "i3", Requires: []string{"x11-display"}}
+}
+
+// §3: a display is resolved on the requiring module's own node.
+func TestAMachineReachRequirementResolvesToTheProviderOnItsOwnNode(t *testing.T) {
+	cat := shelf(windowManager(), displayServer("xorg", "x11-display", DisplayServerSeat))
+	got, err := Resolve(cat, []string{"xorg", "i3"}, workstation(), World{})
+	if err != nil {
+		t.Fatalf("i3 beside xorg did not resolve: %v", err)
+	}
+	if !reflect.DeepEqual(names(got), []string{"xorg", "i3"}) && !reflect.DeepEqual(names(got), []string{"i3", "xorg"}) {
+		t.Errorf("resolved %v", names(got))
+	}
+}
+
+// §3: never answered by installing a provider, and never by another machine's.
+func TestAMachineReachRequirementIsNotPulledInNorAnsweredFromAnotherNode(t *testing.T) {
+	cat := shelf(windowManager(),
+		displayServer("xorg", "x11-display", DisplayServerSeat),
+		displayServer("xwayland", "x11-display"))
+	// Another machine runs xorg and says so to the world; it does not count.
+	world := World{Offered: map[string][]Provider{
+		"x11-display": {{Node: "laptop", At: "laptop.mesh", Module: "xorg"}}}}
+	_, err := Resolve(cat, []string{"i3"}, workstation(), world)
+	if err == nil {
+		t.Fatal("i3 resolved on a machine with no display of its own")
+	}
+	for _, want := range []string{
+		`"x11-display" is wanted by i3`, "usable only on the machine that provides it",
+		"assign one to workstation", "xorg (holds node-display-server)", "xwayland",
+	} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not say %q:\n%v", want, err)
+		}
+	}
+	// With a single provider in the catalogue too: one candidate is still not a choice to make
+	// for somebody, unlike a node-scoped provision without the machine's reach.
+	_, err = Resolve(shelf(windowManager(), displayServer("xorg", "x11-display", DisplayServerSeat)),
+		[]string{"i3"}, workstation(), World{})
+	if err == nil {
+		t.Fatal("xorg was pulled in for i3")
+	}
+	// Not in the first pass, whose refusals take the machine off the network.
+	if _, err := Resolve(cat, []string{"i3"}, workstation(), World{Unchecked: true}); err != nil {
+		t.Errorf("the first pass refused: %v", err)
+	}
+}
+
+func TestTheMachinesReachIsAProvisionsOnlyReachAndIsNodeScoped(t *testing.T) {
+	for _, c := range []struct{ provides, want string }{
+		{`{"name":"x11-display","reach":"internal"}`, `with reach "internal"; a provision's reach is "machine" or nothing`},
+		{`{"name":"x11-display","scope":"mesh","reach":"machine"}`, `at scope "mesh" with the machine's reach`},
+	} {
+		_, err := ParseManifest([]byte(`{"module":"xorg","provides":[` + c.provides + `]}`))
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s: want %q, got %v", c.provides, c.want, err)
+		}
+	}
+	m, err := ParseManifest([]byte(`{"module":"xorg","provides":[{"name":"x11-display","reach":"machine"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !m.Provides[0].MachineReach() {
+		t.Fatal("the reach was not read")
+	}
+	back, _ := json.Marshal(m.Provides[0])
+	if string(back) != `{"name":"x11-display","reach":"machine"}` {
+		t.Errorf("written back as %s", back)
+	}
+}
+
+func TestACatalogueDisagreeingAboutAProvisionsReachIsRefused(t *testing.T) {
+	cat := shelf(windowManager(), displayServer("xorg", "x11-display"),
+		Manifest{Module: "fake-x", Provides: Offers("x11-display")})
+	_, err := Resolve(cat, []string{"xorg", "i3"}, workstation(), World{})
+	if err == nil || !strings.Contains(err.Error(), `the catalogue disagrees about "x11-display"`) {
+		t.Fatalf("a provision with and without the machine's reach gave %v", err)
+	}
+}
+
+// §4: xinitrc and xresources slots, placed by the display server's holder alone.
+func TestTheSessionsFilesArePlacedByTheDisplayServersHolderAlone(t *testing.T) {
+	xorg := Manifest{Module: "xorg", Claims: []Claim{{Name: DisplayServerSeat}},
+		Shell: []ShellCode{{For: "xinitrc", Slot: "first", Code: "xset s off"}},
+		Resources: []map[string]any{
+			{"id": "xinitrc", "type": "file", "path": "/home/op/.xinitrc",
+				"content": "${shell:xinitrc:first}${shell:xinitrc:normal}${shell:xinitrc:last}"},
+			{"id": "xresources", "type": "file", "path": "/home/op/.Xresources",
+				"content": "${shell:xresources:normal}"},
+		}}
+	i3 := Manifest{Module: "i3", Shell: []ShellCode{{For: "xinitrc", Slot: "last", Code: "exec i3"}}}
+	theme := Manifest{Module: "theme", Shell: []ShellCode{
+		{For: "xresources", Slot: "normal", Code: "Xft.dpi: 96"},
+		{For: "zsh", Slot: "normal", Code: "not for the session"},
+	}}
+	r := Resolution{Node: "workstation", Account: "op", Modules: []Manifest{xorg, i3, theme}}
+	out, err := r.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	by := map[string]any{}
+	for _, res := range out {
+		by[res["id"].(string)] = res["content"]
+	}
+	if got := by["xorg.xinitrc"]; got != "# xorg\nxset s off\n# i3\nexec i3\n" {
+		t.Errorf("the .xinitrc is %q", got)
+	}
+	if got := by["xorg.xresources"]; got != "# theme\nXft.dpi: 96\n" {
+		t.Errorf("the .Xresources is %q", got)
+	}
+
+	// The contributions parse; the placeholders parse only in the holder.
+	if _, err := ParseManifest([]byte(`{"module":"i3","shell":[{"for":"xinitrc","slot":"last","code":"exec i3"},` +
+		`{"for":"xresources","slot":"normal","code":"i3.font: x"}]}`)); err != nil {
+		t.Fatalf("a session contribution was refused: %v", err)
+	}
+	for _, c := range []struct{ claims, content, want string }{
+		{``, "${shell:xinitrc:normal}", "does not claim node-display-server; every module's xinitrc is placed by the display server's holder alone"},
+		{`{"name":"node-login-shell"}`, "${shell:xresources:normal}", "does not claim node-display-server"},
+		{`{"name":"node-display-server"}`, "${shell:zsh:normal}", "does not claim node-login-shell"},
+		{`{"name":"node-display-server"}`, "${shell:xsession:normal}", "the session's file one of xinitrc, xresources"},
+	} {
+		raw := `{"module":"holder","claims":[` + c.claims + `],"resources":[{"id":"rc","type":"file","path":"/etc/rc","content":"` +
+			c.content + `"}]}`
+		if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s with claims [%s]: want %q, got %v", c.content, c.claims, c.want, err)
+		}
+	}
+	if _, err := ParseManifest([]byte(`{"module":"xorg","claims":[{"name":"node-display-server"}],` +
+		`"resources":[{"id":"rc","type":"file","path":"/home/op/.xinitrc","content":"${shell:xinitrc:last}"}]}`)); err != nil {
+		t.Errorf("the display server's holder could not place the session's slots: %v", err)
+	}
+}

internal/catalogue/hostname_seat_test.go

@@ -0,0 +1,98 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
+// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
+// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
+// manifest registered before the rename — still holds the one seat.
+
+func withHostnameAlias(t *testing.T) {
+	t.Helper()
+	was := aliases
+	t.Cleanup(func() { aliases = was })
+	UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
+}
+
+func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
+	if _, known := SeatNamed("node-hostname"); !known {
+		t.Fatal("node-hostname is not in the mesh's set")
+	}
+	withHostnameAlias(t)
+	seat, known := SeatNamed("node-hosts-file")
+	if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
+		t.Fatalf("the former name did not resolve: %+v %v", seat, known)
+	}
+	var verbs []string
+	for _, v := range seat.Serves {
+		verbs = append(verbs, v.Name)
+	}
+	if strings.Join(verbs, " ") != "entries add remove" {
+		t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
+	}
+}
+
+// One seat under either name: the module registered before the rename and the one after cannot both
+// hold it on one machine.
+func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
+	withHostnameAlias(t)
+	cat := shelf(
+		mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
+		mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
+	)
+	_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
+	if err == nil || !strings.Contains(err.Error(), "node-hostname") {
+		t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
+	}
+	if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
+		t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
+	}
+}
+
+// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
+// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
+// naming ${machine:name} gives every machine its mesh name.
+func TestTheMachinesNameIsItsSetting(t *testing.T) {
+	cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
+	got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	machines := map[string]string{"ace.internal": "10.42.0.2"}
+	nameOf := func(settings SettingsBy) (string, string) {
+		t.Helper()
+		composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
+			Settings: settings})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if why := composed.LeftOut["hostname"]; why != "" {
+			return "", why
+		}
+		for _, r := range composed.Resources {
+			if r["path"] == "/etc/hostname" {
+				return r["content"].(string), ""
+			}
+		}
+		t.Fatal("no /etc/hostname composed")
+		return "", ""
+	}
+
+	if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
+		t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
+	}
+	if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
+		t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
+	}
+	mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
+	if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
+		t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
+	}
+	node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
+	if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
+		t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
+	}
+}

internal/catalogue/hosts_region_test.go

@@ -1,106 +0,0 @@
-package catalogue_test
-
-import (
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/novox/mesh-controller/internal/catalogue"
-	"github.com/novox/mesh-controller/internal/overlay"
-)
-
-// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
-// through the whole composition, and not only through FactsInto.
-//
-// Composition prefixes a fact's id with the module that asked for it and passes everything else
-// through; a step that dropped `into` on the way would send the region as a whole file, and the
-// host would write the machine's hosts file over again with every unit test above still green.
-
-// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
-// and what the generator writes is not what is under test here.
-type onTheNetwork struct{}
-
-func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
-	return []map[string]any{{"id": "overlay-config", "type": "file",
-		"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
-}
-
-func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
-	shelf := provided(t)
-	// A resolver restarting on the names another module put on the machine, and one resource it
-	// only runs at start — neither of which composition has any business changing.
-	resolver, err := catalogue.ParseManifest([]byte(`{
-		"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
-		"resources": [
-			{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
-			 "content": "seed\n", "at": "start"},
-			{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
-			 "restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
-		]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	shelf[resolver.Module] = resolver
-
-	got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
-		catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
-	out, err := got.Declaration(catalogue.Rendering{
-		Names: names, Machines: names, Suffix: "internal",
-		Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	ids := map[string]map[string]any{}
-	for _, r := range out {
-		ids[r["id"].(string)] = r
-	}
-
-	hosts := ids[overlay.Name+".fact-node-names"]
-	if hosts == nil {
-		t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
-	}
-	if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
-		t.Fatalf("the hosts file is not written into as a region: %v", hosts)
-	}
-	content := hosts["content"].(string)
-	if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
-		t.Errorf("the region does not name the machine:\n%s", content)
-	}
-	for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
-		if strings.Contains(content, floor) {
-			t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
-		}
-	}
-
-	// The resolver's reference to it still names a resource the host will be sent.
-	daemon := ids["resolver.daemon"]
-	if daemon == nil {
-		t.Fatalf("the resolver's service was not composed: %v", keys(ids))
-	}
-	for _, named := range daemon["restart-on"].([]any) {
-		if ids[named.(string)] == nil {
-			t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
-		}
-	}
-	if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
-		t.Errorf("restart-on is %v", daemon["restart-on"])
-	}
-
-	// And a resource's own `at` passes through as the manifest wrote it.
-	if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
-		t.Errorf("a resource's at did not survive composition: %v", seed)
-	}
-}
-
-func keys(m map[string]map[string]any) []string {
-	out := make([]string, 0, len(m))
-	for k := range m {
-		out = append(out, k)
-	}
-	return out
-}