Merge pull request 'The build queue is controlled through the controller and the build seat (hq ADR 0219)' (#57) from feat/the-build-queue-is-controlled into main

cmd/mesh-builder/holder.go

@@ -0,0 +1,386 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go/micro"
+
+	"github.com/novox/mesh-controller/internal/builder"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
+//
+// **The queue is the controller's; the build running here is this machine's.** The controller can
+// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
+// on this machine and containers in this machine's runtime, and only this machine can end them. So
+// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
+// it, pause, resume.
+//
+// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
+// set per build — so "the build running here" is one or none, and kill names it by id so a call
+// that arrives as one build ends and the next begins cannot end the wrong one.
+
+// holder is this machine's state as a holder of the build seat.
+type holder struct {
+	on, seat string
+	// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
+	// rather than silently taking work again.
+	workspace string
+	// say publishes this machine's state: whether it takes work (link.HolderState).
+	say func(link.HolderState) error
+	// remove runs what a kill needs outside the build: the containers left behind.
+	remove builder.Runner
+
+	mu      sync.Mutex
+	paused  bool
+	running *running
+}
+
+// running is the build this machine is doing.
+type running struct {
+	request link.BuildRequest
+	step    string
+	started time.Time
+	cancel  context.CancelFunc
+	killed  bool
+	// returned is the build's own work having ended, before a kill or not; outcome is what was then
+	// announced, and announced whether it went out.
+	returned  bool
+	outcome   string
+	announced bool
+	done      chan struct{}
+}
+
+// pausedFile is where the flag lives in the workspace.
+func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
+
+// newHolder reads the paused flag the workspace keeps.
+func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
+	h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
+	if _, err := os.Stat(pausedFile(workspace)); err == nil {
+		h.paused = true
+	}
+	return h
+}
+
+// Paused is asked by the taking loop before every fetch.
+func (h *holder) Paused() bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.paused
+}
+
+// setPaused records the flag in the workspace first and then in memory, so what this holder says
+// it is and what it would be after a restart never differ.
+func (h *holder) setPaused(paused bool) error {
+	path := pausedFile(h.workspace)
+	if paused {
+		if err := os.MkdirAll(h.workspace, 0o755); err != nil {
+			return err
+		}
+		if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+			return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
+		}
+	} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
+	}
+	h.mu.Lock()
+	h.paused = paused
+	h.mu.Unlock()
+	h.announce()
+	return nil
+}
+
+// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
+// controller reading an older state is a plan read as late rather than a build lost.
+func (h *holder) announce() {
+	if h.say == nil {
+		return
+	}
+	if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
+		fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
+	}
+}
+
+// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
+// a pause outlives the events stream's retention.
+func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
+	h.announce()
+	tick := time.NewTicker(every)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			if h.Paused() {
+				h.announce()
+			}
+		}
+	}
+}
+
+// begin records the build this machine took, with the cancel that ends it.
+func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
+	r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
+	h.mu.Lock()
+	h.running = r
+	h.mu.Unlock()
+	return r
+}
+
+// end clears it, and lets a kill waiting on it know it is over.
+func (h *holder) end(r *running) {
+	h.mu.Lock()
+	if h.running == r {
+		h.running = nil
+	}
+	h.mu.Unlock()
+	close(r.done)
+}
+
+// stepped records the step a build is at, for `current`.
+func (h *holder) stepped(r *running, step string) {
+	h.mu.Lock()
+	r.step = step
+	h.mu.Unlock()
+}
+
+// currentBuild is what `current` answers.
+type currentBuild struct {
+	On      string `json:"on"`
+	Paused  bool   `json:"paused"`
+	Running *struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	} `json:"running,omitempty"`
+	Said string `json:"said"`
+}
+
+func (h *holder) current() currentBuild {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	out := currentBuild{On: h.on, Paused: h.paused}
+	taking := "taking builds"
+	if h.paused {
+		taking = "paused, taking no new build"
+	}
+	if h.running == nil {
+		out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
+		return out
+	}
+	r := h.running
+	elapsed := time.Since(r.started).Round(time.Second)
+	out.Running = &struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
+		r.started.UTC().Format(time.RFC3339), elapsed.String()}
+	out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
+		h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
+	return out
+}
+
+// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
+// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
+// (killAnswer in the controller's queue.go, 75s).
+var (
+	killRemoves = 15 * time.Second
+	killWaits   = 20 * time.Second
+)
+
+// kill ends the build with this id, if it is the one running here and still working: its context
+// cancelled — which kills each command's process group — and the containers it started removed by
+// their label, once at once and again after the build has ended, so one created while it was being
+// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
+// ask so it is not redelivered; the answer says whether that happened.
+func (h *holder) kill(id string) (string, error) {
+	h.mu.Lock()
+	r := h.running
+	if r == nil || r.request.ID != id {
+		doing := "nothing"
+		if r != nil {
+			doing = r.request.ID
+		}
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
+	}
+	if r.returned {
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
+	}
+	r.killed = true
+	cancel, done := r.cancel, r.done
+	h.mu.Unlock()
+
+	cancel()
+	removed, removeErr := h.removeContainers(id)
+	ended := false
+	select {
+	case <-done:
+		ended = true
+	case <-time.After(killWaits):
+	}
+	again, againErr := h.removeContainers(id)
+	removed += again
+	if removeErr == nil {
+		removeErr = againErr
+	}
+	containers := fmt.Sprintf("%d container(s) it started removed", removed)
+	if removeErr != nil {
+		containers = "its containers could not all be listed or removed: " + removeErr.Error()
+	}
+	if !ended {
+		return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
+			"its outcome is in", id, h.on, containers), nil
+	}
+	h.mu.Lock()
+	outcome, announced := r.outcome, r.announced
+	h.mu.Unlock()
+	if !announced {
+		return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
+			"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
+			containers), nil
+	}
+	return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
+		"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
+}
+
+// removeContainers is one pass of removing what the build left, bounded.
+func (h *holder) removeContainers(id string) (int, error) {
+	cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
+	defer stop()
+	return builder.RemoveContainersOf(cleanup, h.remove, id)
+}
+
+// returned records that the build's work ended, and says whether a kill came first — only then is
+// the build killed; an error it ended with on its own is its own outcome.
+func (h *holder) returned(r *running) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	r.returned = true
+	return r.killed
+}
+
+// said records the outcome announced, and whether it went out, for a kill to answer with.
+func (h *holder) said(r *running, outcome string, announced bool) {
+	h.mu.Lock()
+	r.outcome, r.announced = outcome, announced
+	h.mu.Unlock()
+}
+
+// handlers are the seat's verbs, as this machine answers them.
+func (h *holder) handlers() map[string]link.ToolHandler {
+	return map[string]link.ToolHandler{
+		"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
+		"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
+			var args struct {
+				ID string `json:"id"`
+			}
+			if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
+				return nil, errors.New("kill needs the build's id")
+			}
+			said, err := h.kill(strings.TrimSpace(args.ID))
+			if err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": said}, nil
+		},
+		"pause": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(true); err != nil {
+				return nil, err
+			}
+			said := h.on + " is paused: it takes no new build until resumed"
+			if c := h.current(); c.Running != nil {
+				said += "; " + c.Running.ID + " runs on and finishes"
+			}
+			return map[string]any{"said": said, "paused": true}, nil
+		},
+		"resume": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(false); err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
+		},
+	}
+}
+
+func orNothing(s string) string {
+	if s == "" {
+		return "its start"
+	}
+	return s
+}
+
+// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
+// log it would be is the one being ended.
+func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+	}
+	return string(out), nil
+}
+
+// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
+// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
+// the module answering, the seat, scope node, the machine, its description and argument schema — so
+// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
+//
+// One service per machine, named for the seat and identified by the machine, so the answer is this
+// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
+// store, and what it serves is what this binary was built to serve.
+func announcement(seat, module, node string) micro.Info {
+	s, _ := catalogue.SeatNamed(seat)
+	var endpoints []micro.EndpointInfo
+	for _, v := range s.Serves {
+		schema, _ := json.Marshal(v.Input)
+		endpoints = append(endpoints, micro.EndpointInfo{
+			Name:    seat + "__" + v.Name,
+			Subject: link.NodeSeatToolSubject(seat, v.Name, node),
+			// The queue group the verbs are served in, as every runtime announces its own.
+			QueueGroup: "seat." + seat,
+			Metadata: map[string]string{
+				"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
+				"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
+			},
+		})
+	}
+	return micro.Info{
+		ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
+			Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
+		Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
+		Endpoints:   endpoints,
+	}
+}
+
+// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
+func moduleOf(user string) string {
+	if _, module, ok := strings.Cut(user, "."); ok && module != "" {
+		return module
+	}
+	return "build-agent"
+}

cmd/mesh-builder/holder_test.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
+// 0219), and what it says about itself follows.
+func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
+	workspace := t.TempDir()
+	var said []link.HolderState
+	h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
+		said = append(said, s)
+		return nil
+	})
+	if h.Paused() {
+		t.Fatal("a new holder starts paused")
+	}
+	if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
+		t.Fatalf("paused: %v, said %+v", h.Paused(), said)
+	}
+	again := newHolder("ace", link.TheBuildMachine, workspace, nil)
+	if !again.Paused() {
+		t.Fatal("restarted, the holder forgot it was paused")
+	}
+	if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
+		t.Fatal("resumed, the holder came back paused")
+	}
+}
+
+// kill ends the build with that id — its context, which ends its commands — removes what it left by
+// label, and refuses an id it is not building.
+func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	var removed []string
+	h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		removed = append(removed, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\n", nil
+		}
+		return "", nil
+	}
+	kill := h.handlers()["kill"]
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
+		t.Fatal("killed a build while none ran")
+	}
+
+	building, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
+	h.stepped(r, "image")
+	if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
+		t.Fatalf("current says %+v", c)
+	}
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
+		!strings.Contains(err.Error(), "build-1") {
+		t.Fatalf("killed another id: %v", err)
+	}
+	// The build's own goroutine: it ends when its context does, as a build's commands do, and
+	// announces what came of it.
+	go func() {
+		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, true)
+		}
+		h.end(r)
+	}()
+	answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if building.Err() == nil {
+		t.Fatal("the build's context was not cancelled")
+	}
+	if !r.killed {
+		t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
+	}
+	said := answer.(map[string]any)["said"].(string)
+	if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
+		t.Errorf("kill said %q", said)
+	}
+	// Removed at the kill and again once the build had ended: a container made in between is caught.
+	if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
+		t.Errorf("removed %v", removed)
+	}
+	if c := h.current(); c.Running != nil {
+		t.Errorf("after the kill current says %+v", c)
+	}
+}
+
+// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
+func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
+	info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
+	if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
+		t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
+	}
+	kill := info.Endpoints[1]
+	md := kill.Metadata
+	if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
+		md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
+		!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
+		t.Fatalf("kill is announced as %+v", kill)
+	}
+	body, err := json.Marshal(info)
+	if err != nil || len(body) > 8*1024 {
+		t.Fatalf("the answer is %d bytes (%v)", len(body), err)
+	}
+}
+
+// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
+// own error is its outcome. One whose outcome could not be announced is not said to be settled.
+func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
+	_, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
+	if killed := h.returned(r); killed {
+		t.Fatal("a build nobody killed reads as killed")
+	}
+	if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
+		t.Fatalf("killed a build that had ended: %v", err)
+	}
+	h.end(r)
+
+	building, cancel := context.WithCancel(context.Background())
+	r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
+	go func() {
+		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, false)
+		}
+		h.end(r)
+	}()
+	said, err := h.kill("build-2")
+	if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
+		t.Fatalf("kill said %q (%v)", said, err)
+	}
+}

cmd/mesh-builder/main.go

@@ -19,11 +19,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/builder"
@@ -107,48 +109,96 @@ func run() error {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	machine, err := takeWorkFrom(credential, on)
+	js, seat, err := dialFor(credential)
 	if err != nil {
 		return err
 	}
+	defer js.Close()
+
+	// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
+	// paused flag is read from the workspace before anything is taken, so a holder restarted while
+	// paused takes nothing.
+	h := newHolder(on, seat, workspace, func(state link.HolderState) error {
+		body, err := json.Marshal(state)
+		if err != nil {
+			return err
+		}
+		_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
+		return err
+	})
+	if h.Paused() {
+		fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
+	}
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
 	defer machine.Close()
 
+	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
+	// one serves none, and a subscription its holder has no grant for would be refused for ever.
+	if seat == link.TheBuildMachine {
+		stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
+			log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopServing()
+		// And says so, for the console to find (novox/hq ADR 0197).
+		stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
+			announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopAnnouncing()
+		go h.stateWhilePaused(ctx, time.Hour)
+	}
+
 	fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
-		answer(ctx, publisher, on, workspace, work)
+		answer(ctx, publisher, on, workspace, work, h)
 	})
 }
 
-// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
+// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
+// holds.
 //
 // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
 // machine told about both would take work from one and answer on the other, and every log line would
 // say it was fine.
-func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
+func dialFor(credential Credential) (*broker.JetStream, string, error) {
 	// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
 	// A credential for the mesh's bus carries user, password and fingerprint beside the address,
 	// and that is enough to dial it, pinned.
 	if !credential.onTheNewBus() {
-		return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
+		return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
 	}
 	js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
 	// handover): the mesh issues a build machine's credential naming the seat its module claims,
 	// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
 	seat := link.BuildSeatClaimed(credential.seatsClaimed())
 	fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
-	return link.MachineOverNATSOn(js, on, seat), nil
+	return js, seat, nil
 }
 
 // answer does one build and says what happened, whichever way it went.
-func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
+func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
 	request := work.Request()
 
+	// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
+	// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
+	// reaches it through the parent, and that keeps today's meaning below.
+	building, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var mine *running
+	if h != nil {
+		mine = h.begin(request, cancel)
+		defer h.end(mine)
+	}
+
 	// **First thing, and to stdout.** A build request that arrives and produces no visible line until
 	// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
 	// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
@@ -162,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	say := func(step, message string) {
 		fmt.Fprintf(os.Stderr, "  [%s] %s\n", step, message)
 		work.Say(step, message)
+		if mine != nil && step != "run" && step != "output" {
+			h.stepped(mine, step)
+		}
 	}
 	builder.Said = say
 	defer func() { builder.Said = nil }()
@@ -188,11 +241,24 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// The package-registry credential is a build input, so it is resolved before the clone: a
 		// build that could not have resolved its dependencies is refused in front of the reason, not
 		// after a clone that then fails at npm ci.
-		built, err = builder.Build(ctx, builder.Command, publisher,
+		// Every container it starts is labelled with its id, so a kill finds what outlived the
+		// docker client (ADR 0219).
+		built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
 			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
 			forgeFrom(), say, request.Seats)
 	}
-	if err != nil {
+	// Only a build the kill ended: the kill came before its work did. One that finished — built, or
+	// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
+	killed := false
+	if mine != nil {
+		killed = h.returned(mine) && err != nil
+	}
+	if killed {
+		// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
+		// error it ended with is the kill's consequence, not a fault of the source.
+		result.Failed = link.KilledByHand
+		say("failed", link.KilledByHand)
+	} else if err != nil {
 		// A failure is a result. A build that fails and says nothing is indistinguishable from a
 		// builder that is not running, and those want completely different responses.
 		result.Failed = err.Error()
@@ -217,7 +283,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		}
 	}
 
-	if err := work.Announce(ctx, result); err != nil {
+	// A killed build is announced on a context of its own: the build's was the one cancelled, and the
+	// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
+	// to be built again. A machine being stopped is the other case and keeps its meaning: the
+	// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
+	announcing := ctx
+	if killed {
+		fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
+		defer stop()
+		announcing = fresh
+	}
+	announceErr := work.Announce(announcing, result)
+	if mine != nil {
+		h.said(mine, result.Failed, announceErr == nil)
+	}
+	if err := announceErr; err != nil {
 		// Said, not fatal: the build happened. A build reported as failed because announcing it
 		// failed is a lie about work that was done — and the request stays unsettled below only if
 		// nothing was said at all, so another machine can try.

cmd/mesh-controller/build.go

@@ -392,22 +392,34 @@ func buildBehind(ctx context.Context, wait time.Duration) error {
 //
 // Separated from the command so `--behind` can walk a list without a second path to the same act.
 func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
+	_, err := buildOneAsked(ctx, source, path, ref, wait, false)
+	return err
+}
+
+// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
+// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
+// at, never taken in (issue 240), which is what `replay` asks unless told to register.
+func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
+	dryRun bool) (string, error) {
+	if dryRun && wait != 0 {
+		return "", errors.New("a dry run waited for is `build --dry-run`")
+	}
 	// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
 	// the reason, rather than sent to a machine to fail at `git clone`.
 	repository, err := cloneFrom(ctx, source)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	ident, err := openIdentity(ctx)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer ident.Close()
 
 	server, err := connectLink(ctx, nil, nil, nil)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer server.Close()
 
@@ -420,6 +432,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		Ref:        ref,
 		Held:       heldBy(ctx),
 		Seats:      seatBases(ctx),
+		DryRun:     dryRun,
 	}
 	fmt.Printf("asked for %s", source)
 	if source.Seat != "" {
@@ -438,7 +451,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 	seat := buildSeatHeld(ctx)
 	ask, err := askOverOn(seat)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer ask.Close()
 	fmt.Printf("  of %s\n", seat)
@@ -449,26 +462,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		// is still here. A tool call cannot hold a connection for the minutes a build takes; it
 		// follows the build by its id instead.
 		if err := ask.Ask(ctx, request); err != nil {
-			return err
+			return "", err
+		}
+		if dryRun {
+			fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
+				"its outcome is not taken in\n", request.ID)
+			return request.ID, nil
 		}
 		fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
 			"shows what came of it; the module is registered when the outcome comes\n", request.ID)
-		return nil
+		return request.ID, nil
 	}
 
 	result, err := ask.Submit(ctx, request, wait)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 
 	open, err := openStores(ctx)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 	defer open.Close()
 	manifest, kept, err := takeIn(ctx, open.inventory, result)
 	if err != nil {
-		return err
+		return request.ID, err
 	}
 	// Said as recorded: what each artifact is, not where this builder happened to push it.
 	for _, made := range kept.Made {
@@ -478,7 +496,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
 		manifest.Module, manifest.Version, result.On, short(result.Commit))
 	saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
 	fmt.Printf("  run `assign <node> %s` to put it somewhere\n", manifest.Module)
-	return nil
+	return request.ID, nil
 }
 
 // saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
@@ -628,6 +646,8 @@ type answers struct {
 	reported []inventory.Reported
 	// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
 	plans []inventory.Plan
+	// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
+	paused pauseView
 	// refused is why a machine cannot be worked out at all, by name. A different thing from every
 	// other answer here: those are about a machine that was told something, and this is about one
 	// that cannot be told anything — it never reaches waiting, because nothing was computed for it

cmd/mesh-controller/main.go

@@ -73,6 +73,21 @@ func run() error {
 		return askCommand(ctx, args[1:])
 	case "builds":
 		return buildsCommand(ctx, args[1:])
+	// The build queue, controlled by hand (novox/hq ADR 0219).
+	case "queue":
+		return queueCommand(ctx, args[1:])
+	case "cancel":
+		return cancelCommand(ctx, args[1:])
+	case "clear":
+		return clearCommand(ctx, args[1:])
+	case "rebuild":
+		return rebuildCommand(ctx, args[1:])
+	case "replay":
+		return replayCommand(ctx, args[1:])
+	case "kill":
+		return killCommand(ctx, args[1:])
+	case "pause", "resume":
+		return pauseCommand(ctx, args[0], args[1:])
 	case "collection":
 		return collectionCommand(ctx, args[1:])
 	case "plans":
@@ -202,6 +217,14 @@ func usage() {
   build --behind                 build every module the mesh holds older than its source
   build --on <module>            rebuild every module that stands on this module's artifacts, bases first
   builds [<module>]              what has been built lately, and what came of it
+  queue [--json]                 every ask in the build queue: waiting, in flight (where, how long), dead
+  cancel <id>                    drop a waiting or dead ask; recorded failed, cancelled by hand
+  clear [--dead]                 cancel every waiting ask (and the dead ones); never one in flight
+  rebuild <module|build-id>      ask the module's source again, or that build's, under a new id
+  replay <build-id> [--register [--older]]   that build's commit again; a dry run unless --register
+  kill <id>                      end a build where it runs; recorded failed, killed by hand
+  pause [<node>] / resume [<node>]   the build seat's holder there, or every holder, takes nothing new / again
+  plans retry <id>               ask a failed plan's failed builds again, and carry the plan on
   collection [--json]            kept archives held/unheld by a manifest, and what the sweep may let go
   builder issue <name>           a broker account for a build machine, scoped to build work,
                                  delivered as the builder module's broker secret (module add it first)
@@ -271,7 +294,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 	case err != nil && result.Failed != "":
 		fmt.Printf("%s: %v\n", result.ID, err)
 		if result.Module != "" {
-			planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked)
+			planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
 		} else {
 			planFailedBuild(ctx, b.open, result)
 		}
@@ -280,18 +303,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 		// Not a failure: the module is already at what a later request built. A plan that asked
 		// before that later request is answered by it; one that asked after it ignores this.
 		fmt.Printf("%s: %v\n", result.ID, err)
-		planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
+		planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
 		return nil
 	case err != nil:
 		fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
 		if manifest.Module != "" {
-			planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked)
+			planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
 		}
 		return nil
 	}
 	fmt.Printf("%s: %s %s registered, built on %s from %s\n",
 		result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
 	saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
-	planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked)
+	planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
 	return nil
 }

cmd/mesh-controller/plan_retry.go

@@ -0,0 +1,401 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A plan follows what is done to the build queue (novox/hq ADR 0219).
+//
+// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
+//
+//   - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
+//     is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
+//     sends a reader looking for a fault that is a decision. It says what it waits on instead.
+//   - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
+//     failed plan's failed modules and the plan goes on from that tier as if they had built the
+//     first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
+//     beside it — beside it, the plan would either ask it again or stay failed on an outcome the
+//     rebuild has already replaced.
+
+// pauseView is whether the build seat takes work: the holders that said they are paused, and
+// whether that is every holder.
+type pauseView struct {
+	Nodes []string
+	All   bool
+}
+
+// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
+// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
+func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
+	if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
+		return "", false
+	}
+	var asked *time.Time
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
+			if asked == nil || s.AskedAt.Before(*asked) {
+				asked = s.AskedAt
+			}
+		}
+	}
+	if asked == nil {
+		return "", false
+	}
+	waited := now.Sub(*asked)
+	said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
+		strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
+	// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
+	// longer than a day is named.
+	if waited > pausedTooLong {
+		said += " — PAUSED OVER A DAY: `resume` takes builds again"
+	}
+	return said, true
+}
+
+// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
+const pausedTooLong = 24 * time.Hour
+
+// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
+// where the seat being paused changes what a plan says.
+func awaitsABuild(plans []inventory.Plan) bool {
+	for _, p := range plans {
+		if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
+			continue
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if s := p.Modules[m]; s != nil && s.State == "asked" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
+// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
+// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
+// reads as late, which is what it said before this existed.
+func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
+	if !awaitsABuild(plans) {
+		return pauseView{}
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return pauseView{}
+	}
+	seat := buildSeatAmong(entries)
+	holders := holdersAmong(entries, seat)
+	if len(holders) == 0 {
+		return pauseView{}
+	}
+	address, err := broker.BusAddress()
+	if err != nil {
+		return pauseView{}
+	}
+	js, err := broker.Dial(address)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
+		return pauseView{}
+	}
+	defer js.Close()
+	said, err := link.PausedSaid(js, seat, holders)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
+		return pauseView{}
+	}
+	return pauseOf(holders, said)
+}
+
+// pauseOf is the view from what each holder said.
+func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
+	var v pauseView
+	for _, n := range holders {
+		if said[n].Paused {
+			v.Nodes = append(v.Nodes, n)
+		}
+	}
+	sort.Strings(v.Nodes)
+	v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
+	return v
+}
+
+// failedIn is the modules of a plan's current tier that failed to build, sorted.
+func failedIn(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "failed" {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
+// that holds what this one held now (issue 254, ADR 0218).
+func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
+	for _, q := range plans {
+		if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
+			(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
+			continue
+		}
+		return q, true
+	}
+	return inventory.Plan{}, false
+}
+
+// retryRefusal is why a plan cannot be retried, or nothing.
+func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
+	switch {
+	case p.State == inventory.PlanDone:
+		return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
+	case p.State == inventory.PlanSuperseded:
+		return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
+	case p.Open():
+		return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
+	}
+	if len(failedIn(p)) > 0 {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		return nil
+	}
+	stopped := stoppedRollouts(p)
+	if len(stopped) == 0 {
+		return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
+			p.Tier, p.ID, p.Note)
+	}
+	// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
+	// sent — a newer build, and sending this one again would put the older build back on its machines.
+	for _, m := range stopped {
+		if q, found := newerPlanFor(m, p, plans); found {
+			return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
+				"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
+		}
+	}
+	return nil
+}
+
+// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
+// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
+func stoppedRollouts(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
+			len(s.First) > 0 && s.Why != "" {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
+func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
+	for _, q := range plans {
+		if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
+			continue
+		}
+		for _, tier := range q.Tiers {
+			for _, m := range tier {
+				if m == module {
+					return q, true
+				}
+			}
+		}
+	}
+	return inventory.Plan{}, false
+}
+
+// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
+// variable so a test of a retried rollout needs no machine.
+var sendRollout = sendToEach
+
+// resumed sets a failed plan building again once nothing in its tier is failed.
+func resumed(p *inventory.Plan, why string) {
+	if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
+		p.State = inventory.PlanBuilding
+		p.Note = why
+	}
+}
+
+// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
+// that tier: what follows is the plan going on as if they had built the first time.
+func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
+	inv := open.inventory
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return "", err
+	}
+	defer release()
+	p, err := inv.PlanByID(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return "", err
+	}
+	recent, err := inv.RecentPlans(ctx, 50)
+	if err != nil {
+		return "", err
+	}
+	plans = append(plans, recent...)
+	if err := retryRefusal(p, plans); err != nil {
+		return "", err
+	}
+	if len(failedIn(p)) == 0 {
+		return retryRollouts(ctx, open, &p)
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return "", err
+	}
+	byName := map[string]inventory.Entry{}
+	for _, e := range entries {
+		byName[e.Manifest.Module] = e
+	}
+	failed := failedIn(p)
+	var asked []string
+	for _, m := range failed {
+		askModule(ctx, &p, m, byName)
+		if s := p.Modules[m]; s.State == "asked" {
+			asked = append(asked, m+" as "+s.Build)
+		}
+	}
+	resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
+	if err := inv.SavePlan(ctx, p); err != nil {
+		return "", err
+	}
+	if p.State != inventory.PlanBuilding {
+		return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
+	}
+	return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
+		p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
+}
+
+// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
+// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
+func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
+	inv := open.inventory
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return false, "", err
+	}
+	defer release()
+	openPlans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	recent, err := inv.RecentPlans(ctx, 20)
+	if err != nil {
+		return false, "", err
+	}
+	p, found := planHolding(module, openPlans, recent)
+	if !found {
+		return false, "", nil
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	byName := map[string]inventory.Entry{}
+	for _, e := range entries {
+		byName[e.Manifest.Module] = e
+	}
+	was := p.State
+	askModule(ctx, &p, module, byName)
+	s := p.Modules[module]
+	resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
+	if err := inv.SavePlan(ctx, p); err != nil {
+		return false, "", err
+	}
+	if s.State != "asked" {
+		return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
+	}
+	said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
+		s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
+	switch {
+	case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
+		said += "; the plan had failed and builds again from this tier"
+	case was == inventory.PlanFailed:
+		said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
+	}
+	return true, said, nil
+}
+
+// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
+// yet built, else the newest failed plan whose current tier does, and that no open plan of its
+// repository supersedes.
+func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
+	holds := func(p inventory.Plan) bool {
+		if p.Tier >= len(p.Tiers) {
+			return false
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if m == module {
+				s := p.Modules[m]
+				return s == nil || s.State != "built"
+			}
+		}
+		return false
+	}
+	for _, p := range openPlans {
+		if holds(p) {
+			return p, true
+		}
+	}
+	sorted := append([]inventory.Plan(nil), recent...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
+	for _, p := range sorted {
+		if p.State != inventory.PlanFailed || !holds(p) {
+			continue
+		}
+		if _, superseded := newerOpenPlan(p, openPlans); superseded {
+			continue
+		}
+		return p, true
+	}
+	return inventory.Plan{}, false
+}
+
+// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
+// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
+// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
+func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
+	var said []string
+	for _, m := range stoppedRollouts(*p) {
+		s := p.Modules[m]
+		sent, err := sendRollout(ctx, open, s.First)
+		if err != nil {
+			return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
+				m, strings.Join(s.First, ", "), p.ID, err)
+		}
+		now := time.Now().UTC()
+		s.First, s.FirstAt, s.Why = sent, &now, ""
+		said = append(said, m+" to "+strings.Join(sent, ", "))
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
+	if err := open.inventory.SavePlan(ctx, *p); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
+		"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
+}

cmd/mesh-controller/push.go

@@ -1142,6 +1142,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
 	if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
 		return err
 	}
+	// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
+	// whether it was cancelled the moment it took it.
+	if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
+		return err
+	}
 	// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
 	// registration, so a module reading one may watch it before its owner runs anywhere. One that
 	// nothing declares any more is said and kept — what it holds is data.

cmd/mesh-controller/queue.go

@@ -0,0 +1,709 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The build queue, controlled by hand (novox/hq ADR 0219).
+//
+// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
+// may be and never settled — and changed through the controller: an ask cancelled, the queue
+// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
+// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
+//
+// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
+// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
+// rather than waiting for ever on an answer nobody will give.
+
+// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
+// than the holder's worst case (its two passes removing containers and its wait between, 50s).
+const (
+	holderAsks = 15 * time.Second
+	killAnswer = 75 * time.Second
+)
+
+// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
+func dialTheBus() (*broker.JetStream, error) {
+	address, err := broker.BusAddress()
+	if err != nil {
+		return nil, err
+	}
+	js, err := broker.Dial(address)
+	if err != nil {
+		return nil, fmt.Errorf("cannot reach the bus: %w", err)
+	}
+	return js, nil
+}
+
+// queueCommand prints every ask in the build seat's work queue.
+func queueCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("queue", flag.ContinueOnError)
+	asJSON := set.Bool("json", false, "the queue as JSON")
+	if _, err := parseAround(set, args); err != nil {
+		return err
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return err
+	}
+	if *asJSON {
+		body, err := json.MarshalIndent(q, "", "  ")
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(body))
+		return nil
+	}
+	fmt.Print(queueText(q, time.Now()))
+	return nil
+}
+
+// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
+// Never what an ask carries as `held` — that is every artifact the mesh has built.
+func queueText(q link.Queue, now time.Time) string {
+	var b strings.Builder
+	waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
+	fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
+	ago := func(t time.Time) string {
+		if t.IsZero() {
+			return "asked at an unknown time"
+		}
+		return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
+	}
+	what := func(a link.QueuedAsk) string {
+		s := a.Repository
+		if a.Path != "" {
+			s += " at " + a.Path
+		}
+		if a.Ref != "" {
+			s += " on " + a.Ref
+		}
+		return s
+	}
+	if len(running) > 0 {
+		fmt.Fprintln(&b, "\nin flight:")
+		for _, a := range running {
+			where := "taken, not yet said where"
+			switch {
+			case a.On != "":
+				where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
+			case a.Was != "":
+				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
+			}
+			fmt.Fprintf(&b, "  %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
+		}
+	}
+	if len(waiting) > 0 {
+		fmt.Fprintln(&b, "\nwaiting:")
+		for _, a := range waiting {
+			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+		}
+	}
+	if len(dead) > 0 {
+		fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
+		for _, a := range dead {
+			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+		}
+	}
+	switch {
+	case len(q.Asks) == 0:
+		fmt.Fprintln(&b, "nothing is asked of it")
+	default:
+		fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
+	}
+	return b.String()
+}
+
+// cancelCommand drops one waiting or dead ask.
+func cancelCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("cancel <build id>")
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	seat := buildSeatHeld(ctx)
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return err
+	}
+	ask, found := q.Find(args[0])
+	if !found {
+		return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
+			"what came of it", args[0], seat)
+	}
+	said, err := cancelAsk(ctx, js, open, seat, ask)
+	if err != nil {
+		return err
+	}
+	fmt.Println(said)
+	return nil
+}
+
+// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
+//
+// **In this order, and each step for a reason** (novox/hq ADR 0219):
+//  1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
+//     is running — that is `kill`, on the machine running it;
+//  2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
+//  3. for a waiting ask, the worker is read again: one handed out since the queue was read was
+//     taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
+//     read the mark first and ends it as cancelled, or is building it;
+//  4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
+//     handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
+//     at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
+//     that took it before the mark is building it, and only `kill` ends that;
+//  5. the message is deleted by its sequence;
+//  6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
+func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
+	if ask.State == link.AskInFlight && ask.On != "" {
+		return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
+			"ends the build where it runs", ask.ID, ask.On, ask.ID)
+	}
+	if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
+		return "", err
+	}
+	withdraw := func() {
+		if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
+			fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
+		}
+	}
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	switch ask.State {
+	case link.AskWaiting:
+		if taken, err := takenSince(js, worker, ask.Seq); err != nil {
+			withdraw()
+			return "", err
+		} else if taken {
+			withdraw()
+			return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
+				"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
+				"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
+		}
+	case link.AskInFlight:
+		if started, err := startedSince(ctx, js, seat, ask); err != nil {
+			withdraw()
+			return "", err
+		} else if started != "" {
+			withdraw()
+			return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
+				"ends it there", ask.ID, started, ask.ID)
+		}
+	}
+	if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
+		!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
+		return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
+			"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
+	}
+	recordCancelled(ctx, open, ask)
+	return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
+		"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
+}
+
+// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
+// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
+var holderLooks = 5 * time.Second
+
+// startedSince waits out a holder's look at the cancelled set and says the machine that started the
+// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
+// a start of a build.
+func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
+	select {
+	case <-ctx.Done():
+		return "", ctx.Err()
+	case <-time.After(holderLooks):
+	}
+	since := time.Now().Add(-7 * 24 * time.Hour)
+	if !ask.AskedAt.IsZero() {
+		since = ask.AskedAt.Add(-time.Minute)
+	}
+	heard, err := link.ReadBuildEvents(ctx, js, seat, since)
+	if err != nil {
+		return "", err
+	}
+	s, ok := heard.Started[ask.ID]
+	if !ok || heard.Outcomes[ask.ID] {
+		return "", nil
+	}
+	at, _ := time.Parse(time.RFC3339Nano, s.At)
+	if ask.Started.IsZero() || at.After(ask.Started) {
+		return s.On, nil
+	}
+	return "", nil
+}
+
+// takenSince is whether the worker has handed out the ask at this sequence.
+func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
+	info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
+	if errors.Is(err, nats.ErrConsumerNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
+	}
+	return info.Delivered.Stream >= seq, nil
+}
+
+// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
+// then the plan that asked for it — by the id it asked with, or by repository and path.
+func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
+	r := ask.Request
+	result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
+		Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
+	_ = builds{open.inventory, open}.Built(ctx, result)
+}
+
+// clearCommand cancels every waiting ask, and with --dead every dead one too.
+func clearCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("clear", flag.ContinueOnError)
+	dead := set.Bool("dead", false, "the dead asks too")
+	if _, err := parseAround(set, args); err != nil {
+		return err
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	seat := buildSeatHeld(ctx)
+	said, err := clearQueue(ctx, js, open, seat, *dead)
+	fmt.Print(said)
+	return err
+}
+
+// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
+func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
+	q, err := link.ReadQueue(ctx, js, seat)
+	if err != nil {
+		return "", err
+	}
+	var b strings.Builder
+	cancelled, refused := 0, 0
+	for _, a := range q.Asks {
+		if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
+			continue
+		}
+		said, err := cancelAsk(ctx, js, open, seat, a)
+		if err != nil {
+			refused++
+			fmt.Fprintf(&b, "  %s: %v\n", a.ID, err)
+			continue
+		}
+		cancelled++
+		fmt.Fprintf(&b, "  %s\n", said)
+	}
+	what := "waiting"
+	if dead {
+		what = "waiting and dead"
+	}
+	fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
+	if refused > 0 {
+		fmt.Fprintf(&b, ", %d could not be", refused)
+	}
+	fmt.Fprintln(&b)
+	if n := len(q.Of(link.AskInFlight)); n > 0 {
+		fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
+	}
+	if n := len(q.Of(link.AskDead)); n > 0 && !dead {
+		fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
+	}
+	if refused > 0 {
+		return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
+	}
+	return b.String(), nil
+}
+
+// rebuildCommand asks the module's current source again — or, given a build's id, that build's
+// repository, path and ref — under a new id.
+func rebuildCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("rebuild <module | build id>")
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	var source buildSource
+	var path, ref, module string
+	if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
+		return err
+	} else if found {
+		source, module = sourceOfBuild(b, entries)
+		path, ref = b.Path, b.Ref
+		// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
+		// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
+		// that commit out over everything since. Building that commit again is `replay`, which says
+		// what it would do and refuses to register it while anything newer is asked or registered.
+		if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
+			ref = followedBranch(e.Source.Ref)
+			follows := ref
+			if follows == "" {
+				follows = "the repository's default branch"
+			}
+			fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
+				"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
+		}
+	} else if e, known := entryNamed(entries, args[0]); known {
+		// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
+		source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
+		path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
+	} else {
+		return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
+	}
+
+	// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
+	// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
+	if module != "" {
+		joined, said, err := joinAPlan(ctx, open, module)
+		if err != nil {
+			return err
+		}
+		if joined {
+			fmt.Println(said)
+			return nil
+		}
+	}
+	id, err := askABuild(ctx, source, path, ref)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("rebuild asked as %s\n", id)
+	return nil
+}
+
+// entryNamed is the catalogue's entry for a module.
+func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
+	for _, e := range entries {
+		if e.Manifest.Module == name {
+			return e, true
+		}
+	}
+	return inventory.Entry{}, false
+}
+
+// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
+// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
+// (ADR 0111) — else the repository as it was cloned.
+func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
+	for _, e := range entries {
+		if (b.Module != "" && e.Manifest.Module == b.Module) ||
+			(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
+			return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
+		}
+	}
+	return buildSource{Repository: b.Repository}, b.Module
+}
+
+// replayCommand asks a recorded build's repository and path again at the commit it built.
+func replayCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("replay", flag.ContinueOnError)
+	register := set.Bool("register", false, "register what it builds, as any build is")
+	older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) != 1 {
+		return errors.New("replay <build id> [--register [--older]]")
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	b, found, err := inv.BuildByID(ctx, positionals[0])
+	if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("no build %s is recorded", positionals[0])
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	source, module := sourceOfBuild(b, entries)
+	var history []inventory.Build
+	if module != "" {
+		if history, err = inv.Builds(ctx, module, 100); err != nil {
+			return err
+		}
+	}
+	// What is asked of the module and not yet answered, when the replay would be registered.
+	var outstanding []string
+	if *register {
+		js, err := dialTheBus()
+		if err != nil {
+			return err
+		}
+		q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
+		js.Close()
+		if err != nil {
+			return err
+		}
+		plans, err := inv.OpenPlans(ctx)
+		if err != nil {
+			return err
+		}
+		outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
+	}
+	if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
+		return err
+	}
+	id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
+	if err != nil {
+		return err
+	}
+	fmt.Println(replaySaid(b, module, id, *register))
+	return nil
+}
+
+// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
+//
+// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
+// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
+// older commit is newer than everything since, and would roll that older commit out as the module's
+// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
+// --register says otherwise, and --register is refused when a newer build of a different commit is
+// registered, unless --older says that is the point.
+//
+// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
+// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
+// and their builds — made from newer source — would be recorded and never registered (issue 219
+// orders by ask), the plan waiting on them sending the older commit instead.
+func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
+	outstanding []string) error {
+	if b.Commit == "" {
+		return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
+			"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
+	}
+	if older && !register {
+		return errors.New("--older only says what --register may do; a dry run registers nothing")
+	}
+	if register && len(outstanding) > 0 {
+		return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
+			"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
+			orNone(module), strings.Join(outstanding, "; "), b.ID)
+	}
+	if !register || older {
+		return nil
+	}
+	for _, h := range history {
+		if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
+			continue
+		}
+		if h.AskedOrAt().After(b.AskedOrAt()) {
+			return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
+				"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
+				"without --register looks at it; --register --older registers it anyway",
+				module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
+		}
+	}
+	return nil
+}
+
+// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
+func replaySaid(b inventory.Build, module, id string, register bool) string {
+	what := b.Repository
+	if module != "" {
+		what = module
+	}
+	said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
+	if !register {
+		return said + "\n  a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
+			"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
+	}
+	return said + "\n  registered when it is built, as the module's current version — newer than every build " +
+		"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
+}
+
+// killCommand finds the machine running a build and asks its holder to end it.
+func killCommand(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("kill <build id>")
+	}
+	id := args[0]
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	since := time.Now().Add(-7 * 24 * time.Hour)
+	if at, ok := link.BuildAskedAt(id); ok {
+		since = at.Add(-time.Minute)
+	}
+	heard, err := link.ReadBuildEvents(ctx, js, seat, since)
+	if err != nil {
+		return err
+	}
+	started, ok := heard.Started[id]
+	if !ok {
+		return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
+	}
+	if heard.Outcomes[id] {
+		return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
+	}
+	answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
+	if err != nil {
+		return err
+	}
+	return printHolderAnswer(started.On, answer)
+}
+
+// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
+func pauseCommand(ctx context.Context, verb string, args []string) error {
+	if len(args) > 1 {
+		return fmt.Errorf("%s [node]", verb)
+	}
+	js, err := dialTheBus()
+	if err != nil {
+		return err
+	}
+	defer js.Close()
+	seat := buildSeatHeld(ctx)
+	nodes := args
+	if len(nodes) == 0 {
+		if nodes, err = buildSeatHolders(ctx, seat); err != nil {
+			return err
+		}
+		if len(nodes) == 0 {
+			return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
+		}
+	}
+	failed := 0
+	for _, node := range nodes {
+		answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
+		if err != nil {
+			fmt.Printf("%s: %v\n", node, err)
+			failed++
+			continue
+		}
+		if err := printHolderAnswer(node, answer); err != nil {
+			failed++
+		}
+	}
+	if failed > 0 {
+		return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
+	}
+	return nil
+}
+
+// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
+func printHolderAnswer(node string, answer link.Answer) error {
+	if answer.Error != "" {
+		fmt.Printf("%s: %s\n", node, answer.Error)
+		return errors.New(answer.Error)
+	}
+	var said struct {
+		Said string `json:"said"`
+	}
+	if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
+		fmt.Printf("%s: %s\n", node, said.Said)
+		return nil
+	}
+	fmt.Printf("%s: %s\n", node, string(answer.Result))
+	return nil
+}
+
+// buildSeatHolders is every machine an assigned module holding the build seat runs on.
+func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
+	open, err := openStores(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer open.Close()
+	entries, err := open.inventory.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return holdersAmong(entries, seat), nil
+}
+
+// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
+func holdersAmong(entries []inventory.Entry, seat string) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, e := range entries {
+		if !e.Manifest.ClaimsSeat(seat) {
+			continue
+		}
+		for _, n := range e.On {
+			if !seen[n] {
+				seen[n] = true
+				out = append(out, n)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
+// by repository and path, and every open plan holding it not yet built.
+func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
+	var out []string
+	for _, a := range q.Asks {
+		if repositoryMatches(a.Repository, repository) && a.Path == path {
+			out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
+		}
+	}
+	if module == "" {
+		return out
+	}
+	for _, p := range plans {
+		if !p.Open() {
+			continue
+		}
+		for _, tier := range p.Tiers {
+			for _, m := range tier {
+				if m == module {
+					if st := p.Modules[m]; st == nil || st.State != "built" {
+						out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
+					}
+				}
+			}
+		}
+	}
+	return out
+}

cmd/mesh-controller/queue_test.go

@@ -0,0 +1,772 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
+//
+//	docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
+//	make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
+//	MESH_TEST_NATS=nats://127.0.0.1:14294 \
+//	MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
+//	  go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
+
+// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
+func asksRecorded(t *testing.T) *[]string {
+	t.Helper()
+	var asked []string
+	was := askABuild
+	askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
+		id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
+		asked = append(asked, source.Repository+"#"+id)
+		return id, nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	return &asked
+}
+
+// twoTiers registers two modules, b standing on a, each with a source a plan asks.
+func twoTiers(t *testing.T, open *stores) {
+	t.Helper()
+	for _, name := range []string{"a", "b"} {
+		if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
+			inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// A failed plan is retried: its failed module asked again under a new id, the plan building at that
+// tier, and when that build comes in the plan goes on and asks its next tier.
+func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	before := time.Now().UTC().Add(-time.Hour)
+	failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
+		Note: "a failed to build in tier 0",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
+			Why: link.KilledByHand}}}
+	if err := open.inventory.SavePlan(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+
+	said, err := retryPlan(ctx, open, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, err := open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := p.Modules["a"]
+	if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
+		t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
+	}
+	if !strings.Contains(said, a.Build) {
+		t.Errorf("retry does not say the new id: %q", said)
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+
+	// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
+	planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
+	if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
+		t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
+	}
+	asking, _ := link.BuildAskedAt(a.Build)
+	planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
+	p, err = open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
+		t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
+	}
+	if len(*asked) != 2 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// What retry refuses, and says why.
+func TestRetryRefusesWhatItCannotResume(t *testing.T) {
+	at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	plan := func(id, state string, created time.Time) inventory.Plan {
+		return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
+			Created: created, State: state, Tiers: [][]string{{"a"}},
+			Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
+	}
+	failed := plan("plan-1", inventory.PlanFailed, at)
+	for _, c := range []struct {
+		p      inventory.Plan
+		others []inventory.Plan
+		says   string
+	}{
+		{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
+		{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
+		{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
+		{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
+	} {
+		err := retryRefusal(c.p, c.others)
+		if err == nil || !strings.Contains(err.Error(), c.says) {
+			t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
+		}
+	}
+	stopped := failed
+	stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
+	stopped.Note = "a stopped at its first machine"
+	if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
+		t.Errorf("a plan with nothing failed to build was retried: %v", err)
+	}
+	// Another branch's newer plan, an older one, and a failed one do not supersede it.
+	other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
+	other.Branch = "release"
+	if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
+		plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
+		t.Errorf("refused for a plan that does not supersede it: %v", err)
+	}
+}
+
+// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
+func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	before := time.Now().UTC().Add(-time.Hour)
+	failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
+		Note:    "a failed to build in tier 0",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
+	if err := open.inventory.SavePlan(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"a"}); err != nil {
+		t.Fatal(err)
+	}
+	p, err := open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
+		t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+	// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
+	if err := rebuildCommand(ctx, []string{"b"}); err != nil {
+		t.Fatal(err)
+	}
+	if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
+		t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
+	}
+	if len(*asked) != 2 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
+func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
+	at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
+		Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
+	newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
+		State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
+	if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
+		t.Fatal("joined a failed plan a newer open one supersedes")
+	}
+	if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
+		t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
+	}
+	built := failed
+	built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
+	if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
+		t.Fatal("joined a plan that has the module built")
+	}
+}
+
+// replay is a dry run unless registered, and registering an older commit than one registered since
+// is refused unless --older says it is meant (novox/hq issue 207).
+func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
+	asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
+	newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
+	history := []inventory.Build{newer, old}
+
+	if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
+		t.Errorf("a dry run was refused: %v", err)
+	}
+	err := replayRefusal(old, "a", history, true, false, nil)
+	if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
+		t.Errorf("registering an older commit than the one registered was not refused: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
+		t.Errorf("--register --older was refused: %v", err)
+	}
+	if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
+		t.Errorf("registering the newest build again was refused: %v", err)
+	}
+	// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
+	same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
+	broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
+	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
+		t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
+	}
+	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
+		t.Error("a build that recorded no commit was replayed")
+	}
+	if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
+		t.Error("--older without --register was taken")
+	}
+	// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
+	// holding it unbuilt, would be replaced by a replay asked now (issue 219).
+	q := link.Queue{Asks: []link.QueuedAsk{
+		{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
+		{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
+		{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
+	}}
+	plans := []inventory.Plan{
+		{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
+		{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
+		{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
+	}
+	outstanding := outstandingFor("a", "novox/a", "", q, plans)
+	if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
+		t.Fatalf("outstanding: %v", outstanding)
+	}
+	if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
+		t.Errorf("registered over an outstanding ask: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
+		t.Errorf("a dry run was refused for what is outstanding: %v", err)
+	}
+	if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
+		t.Errorf("a dry replay does not say what it is: %q", said)
+	}
+	if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
+		t.Errorf("a registered replay does not say what it does: %q", said)
+	}
+}
+
+// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
+// paused on some holders only is not a reason the plan is waiting.
+func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
+	now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
+	asked := now.Add(-12 * time.Minute)
+	p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
+		Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
+
+	all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
+	line := planLineWith(p, now, all)
+	if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan on a paused seat reads %q", line)
+	}
+	if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
+		t.Errorf("status --json says %+v", st)
+	}
+	if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
+		t.Errorf("a plan waiting on a paused seat counted late")
+	}
+
+	longAgo := now.Add(-25 * time.Hour)
+	forgotten := p
+	forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
+	if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan paused over a day reads %q", line)
+	}
+
+	some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
+	if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
+		t.Fatalf("%+v", some)
+	}
+	if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
+		t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
+	}
+	if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
+		t.Errorf("status --json: %+v", st)
+	}
+	// A plan rolling out, or with nothing asked, is not waiting on the seat.
+	rolling := p
+	rolling.State = inventory.PlanRolling
+	if _, paused := pausedWaiting(rolling, all, now); paused {
+		t.Error("a rolling plan reads as waiting on the build seat")
+	}
+}
+
+// The queue's verbs are the controller seat's, each to the command it names.
+func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
+	for _, c := range []struct {
+		verb string
+		args map[string]any
+		want []string
+	}{
+		{"queue", nil, []string{"queue"}},
+		{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
+		{"clear", nil, []string{"clear"}},
+		{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
+		{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
+		{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
+		{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
+		{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
+		{"pause", nil, []string{"pause"}},
+		{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
+		{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
+	} {
+		got, err := argvFor(c.verb, c.args)
+		if err != nil || !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
+		}
+	}
+	if _, err := argvFor("cancel", nil); err == nil {
+		t.Error("cancel without an id was taken")
+	}
+	declared := map[string]bool{}
+	for _, v := range catalogue.ControllerVerbs {
+		declared[v.Name] = true
+	}
+	for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
+		if !declared[v] {
+			t.Errorf("%s is not a verb of the controller seat", v)
+		}
+	}
+}
+
+// --- against a real bus -----------------------------------------------------------------------
+
+// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
+// pointed at it, and a function that asks the seat one build.
+func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
+	t.Helper()
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	t.Setenv(broker.NATSVar, url)
+	js, err := broker.Dial(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
+		Emits: []string{"started", "built", "log.*", "paused.*"}}
+	if err := broker.AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
+	if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
+		link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
+		_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
+		_ = js.Context().PurgeStream(broker.EventsStream)
+	})
+	ask := func(id, repository string) link.BuildRequest {
+		r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
+		body, _ := json.Marshal(r)
+		if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
+			t.Fatal(err)
+		}
+		return r
+	}
+	return js, ask
+}
+
+// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
+func deadOne(t *testing.T, js *broker.JetStream) {
+	t.Helper()
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	for i := 0; i < worker.MaxDeliver; i++ {
+		msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
+		if err != nil {
+			t.Fatalf("delivery %d: %v", i+1, err)
+		}
+		_ = msgs[0].Nak()
+	}
+	// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
+	// then, and stops counting it pending.
+	if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
+		t.Fatalf("an ask past its deliveries was delivered again")
+	}
+}
+
+// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
+// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
+// is still found.
+func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	twoTiers(t, open)
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	asked, _ := link.BuildAskedAt(id)
+	plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
+		t.Fatalf("the queue reads %+v", q)
+	}
+	if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
+		strings.Contains(text, "secret-ish") {
+		t.Errorf("the queue says:\n%s", text)
+	}
+
+	if err := cancelCommand(ctx, []string{id}); err != nil {
+		t.Fatal(err)
+	}
+	if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
+		t.Fatalf("the ask is still queued: %+v", q.Asks)
+	}
+	if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
+		t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
+	}
+	b, found, err := open.inventory.BuildByID(ctx, id)
+	if err != nil || !found || b.Failed != link.CancelledByHand {
+		t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
+	}
+	p, err := open.inventory.PlanByID(ctx, plan.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
+		t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
+	}
+	if err := cancelCommand(ctx, []string{id}); err == nil {
+		t.Error("an ask cancelled already was cancelled again")
+	}
+}
+
+// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
+func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	start := time.Now()
+	dead := link.NewBuildID(start)
+	ask(dead, "https://forge.example/novox/dead.git")
+	deadOne(t, js)
+	var waiting []string
+	for i := 1; i <= 2; i++ {
+		id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
+		waiting = append(waiting, id)
+		ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
+	}
+
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
+		t.Fatalf("the queue reads %+v", q)
+	}
+	said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
+		t.Errorf("clear said:\n%s", said)
+	}
+	q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
+		t.Fatalf("after clear the queue is %+v", q.Asks)
+	}
+	if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
+		t.Fatal(err)
+	}
+	if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
+		t.Fatalf("clear --dead left %+v", q.Asks)
+	}
+	for _, id := range append(waiting, dead) {
+		if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
+			t.Errorf("%s is recorded as %+v", id, b)
+		}
+	}
+}
+
+// An ask in flight is not cancelled: kill ends it where it runs.
+func TestCancelRefusesAnAskInFlight(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+	if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "ace" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
+	if err == nil || !strings.Contains(err.Error(), "kill "+id) {
+		t.Fatalf("an ask in flight was cancelled: %v", err)
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a refused cancel recorded an outcome")
+	}
+}
+
+// kill finds the machine from the build's start and asks that machine's holder; pause asks the
+// machine named. Each prints what the holder answered, and a refusal is the command's failure.
+func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
+	js, _ := aBuildQueue(t)
+	ctx := t.Context()
+	asked := map[string]string{}
+	handlers := map[string]link.ToolHandler{
+		"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
+			var args struct{ ID string }
+			_ = json.Unmarshal(raw, &args)
+			asked["kill"] = args.ID
+			if args.ID != "build-running" {
+				return nil, fmt.Errorf("ace is not building %s", args.ID)
+			}
+			return map[string]any{"said": "killed " + args.ID}, nil
+		},
+		"pause": func(context.Context, json.RawMessage) (any, error) {
+			asked["pause"] = "ace"
+			return map[string]any{"said": "ace is paused"}, nil
+		},
+	}
+	stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stop()
+	for _, id := range []string{"build-running", "build-other"} {
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := killCommand(ctx, []string{"build-running"}); err != nil {
+		t.Fatal(err)
+	}
+	if asked["kill"] != "build-running" {
+		t.Fatalf("the holder was asked %v", asked)
+	}
+	if err := killCommand(ctx, []string{"build-other"}); err == nil {
+		t.Error("the holder's refusal was not the command's")
+	}
+	if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
+		t.Errorf("a build nobody started: %v", err)
+	}
+	if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
+		t.Fatalf("pause: %v %v", err, asked)
+	}
+	if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
+		t.Error("a machine nothing answers on was resumed")
+	}
+}
+
+// A plan that stopped at its first machine is retried: the module sent to that machine again, the
+// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
+func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var sentTo [][]string
+	was := sendRollout
+	sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
+		sentTo = append(sentTo, names)
+		return names, nil
+	}
+	t.Cleanup(func() { sendRollout = was })
+
+	long := time.Now().UTC().Add(-2 * time.Hour)
+	stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
+		Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
+		Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
+			First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
+	if err := open.inventory.SavePlan(ctx, stopped); err != nil {
+		t.Fatal(err)
+	}
+
+	// A newer plan holding a refuses it: sending the older build would put it back.
+	newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
+		State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
+	if err := open.inventory.SavePlan(ctx, newer); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
+		t.Fatalf("retried under a newer plan: %v", err)
+	}
+	newer.State = inventory.PlanSuperseded
+	if err := open.inventory.SavePlan(ctx, newer); err != nil {
+		t.Fatal(err)
+	}
+
+	said, err := retryPlan(ctx, open, stopped.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
+		t.Fatalf("sent %v; said %q", sentTo, said)
+	}
+	p, err := open.inventory.PlanByID(ctx, stopped.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := p.Modules["a"]
+	if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
+		t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
+	}
+	// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
+	advancePlans(ctx, open)
+	if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
+		t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("asked %v", *asked)
+	}
+}
+
+// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
+// the plan, asked later, never answers it (novox/hq ADR 0219).
+func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := time.Now().UTC().Add(-time.Minute)
+	plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+	planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
+	p, _ := open.inventory.PlanByID(ctx, plan.ID)
+	if p.Modules["a"].State != "asked" {
+		t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
+	}
+	// From the records too: a later build of the module recorded is not the plan's.
+	recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
+	if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
+		t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
+	}
+	planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
+	if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
+		t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
+	}
+}
+
+// rebuild of a build made at a commit asks what the module follows now, never the commit.
+func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var refs []string
+	was := askABuild
+	askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
+		refs = append(refs, ref)
+		return was(c, source, path, ref)
+	}
+	if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
+		Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
+		t.Fatalf("asked %v at %v", *asked, refs)
+	}
+}
+
+// An ask the worker counts out that no machine said it started is cancelled only if no start comes
+// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
+func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	was := holderLooks
+	holderLooks = 300 * time.Millisecond
+	t.Cleanup(func() { holderLooks = was })
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	// The holder that took it says it started, while the cancel waits.
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		_, _ = js.Context().Publish(link.BuildStarted(), started)
+	}()
+	if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
+		t.Fatalf("a build that started was cancelled: %v", err)
+	}
+	if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
+		t.Error("the withdrawn cancel is still marked")
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a withdrawn cancel recorded an outcome")
+	}
+}

cmd/mesh-controller/readable.go

@@ -176,7 +176,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
 	out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
 		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
 		Reported: []machineReported{}, Unresolved: []machineUnresolved{},
-		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())}
+		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
 	// In a stated order, so two readings of an unchanged mesh are the same document.
 	untakenNodes := make([]string, 0, len(asked.untaken))
 	for name := range asked.untaken {

cmd/mesh-controller/release_plan.go

@@ -324,37 +324,52 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 	for _, e := range entries {
 		byName[e.Manifest.Module] = e
 	}
-	now := time.Now().UTC()
 	for _, name := range p.Tiers[p.Tier] {
-		state := p.Modules[name]
-		if state == nil {
-			state = &inventory.PlanModule{}
-			p.Modules[name] = state
-		}
-		e, known := byName[name]
-		if !known {
-			state.State = "failed"
-			state.Why = "no longer in the catalogue"
-			p.State = inventory.PlanFailed
-			p.Note = name + " is no longer in the catalogue"
-			continue
-		}
-		source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
-		fmt.Printf("  tier %d: ", p.Tier)
-		// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
-		if err := buildOne(ctx, source, e.Source.Path, followedBranch(e.Source.Ref), 0); err != nil {
-			state.State = "failed"
-			state.Why = err.Error()
-			p.State = inventory.PlanFailed
-			p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
-			continue
-		}
-		state.State = "asked"
-		state.AskedAt = &now
+		askModule(ctx, p, name, byName)
 	}
 	return nil
 }
 
+// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
+// variable so a test of what a plan does around an ask needs no build machine.
+var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
+	return buildOneAsked(ctx, source, path, ref, 0, false)
+}
+
+// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
+// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
+func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
+	now := time.Now().UTC()
+	state := p.Modules[name]
+	if state == nil {
+		state = &inventory.PlanModule{}
+		p.Modules[name] = state
+	}
+	e, known := byName[name]
+	if !known {
+		state.State = "failed"
+		state.Why = "no longer in the catalogue"
+		p.State = inventory.PlanFailed
+		p.Note = name + " is no longer in the catalogue"
+		return
+	}
+	source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
+	fmt.Printf("  tier %d: ", p.Tier)
+	// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
+	id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
+	if err != nil {
+		state.State = "failed"
+		state.Why = err.Error()
+		p.State = inventory.PlanFailed
+		p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
+		return
+	}
+	state.State = "asked"
+	state.AskedAt = &now
+	state.Build = id
+	state.Why, state.Commit, state.BuiltAt = "", "", nil
+}
+
 // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
 // advances what that completes. Called from the daemon's take-in of every outcome.
 //
@@ -363,7 +378,7 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 // the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
 // would send machines, and the next tier, what the later merge replaced. asked is zero when the
 // build's request time is not known, and such an outcome is taken as before.
-func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) {
+func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
 	inv := open.inventory
 	// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
 	// than write over what the holder is about to save. Not taken, it is still in the build records,
@@ -399,7 +414,17 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			state = &inventory.PlanModule{}
 			p.Modules[module] = state
 		}
-		if askedBefore(asked, state.AskedAt) {
+		// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
+		// is, as before, when it was asked at or after the plan's ask (issue 219).
+		// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
+		// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
+		// build, made from other source, and settling the plan with it would send that. A plan from
+		// before ids were kept is matched as it was: by when the build was asked (issue 219).
+		if state.Build != "" {
+			if state.Build != id {
+				continue
+			}
+		} else if askedBefore(asked, state.AskedAt) {
 			continue
 		}
 		if failed != "" {
@@ -523,6 +548,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
 	// outcome, whoever was listening.
 	recorded := map[string][]inventory.Build{}
+	byID := map[string]inventory.Build{}
 	for _, m := range tier {
 		if s := p.Modules[m]; s != nil && s.State == "asked" {
 			builds, err := inv.Builds(ctx, m, 5)
@@ -530,9 +556,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 				return false, err
 			}
 			recorded[m] = builds
+			// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
+			if s.Build != "" {
+				b, found, err := inv.BuildByID(ctx, s.Build)
+				if err != nil {
+					return false, err
+				}
+				if found {
+					byID[s.Build] = b
+				}
+			}
 		}
 	}
-	if settleFromRecords(p, tier, recorded) {
+	if settleFromRecords(p, tier, recorded, byID) {
 		return true, nil
 	}
 	// Asked: wait for every build.
@@ -811,7 +847,11 @@ func planTicker(ctx context.Context, open *stores) {
 }
 
 // planLine is one plan as `status` says it.
-func planLine(p inventory.Plan, now time.Time) string {
+func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
+
+// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
+// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
+func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
 	where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
 	switch p.State {
 	case inventory.PlanDone:
@@ -822,6 +862,9 @@ func planLine(p inventory.Plan, now time.Time) string {
 		return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
 	}
 	since := now.Sub(p.Updated).Round(time.Second)
+	if waiting, paused := pausedWaiting(p, pause, now); paused {
+		return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
+	}
 	late := ""
 	if since > planWaitBound {
 		late = " — LATE"
@@ -835,20 +878,49 @@ func planLine(p inventory.Plan, now time.Time) string {
 
 // planFailedBuild marks the module a failed build was for when the result names no module: by the
 // repository and path the plan's modules were asked at.
+//
+// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an
+// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched
+// to the module it was asked for exactly. Repository and path remain for a plan from before ids
+// were kept.
 func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
+	asked, _ := link.BuildAskedAt(result.ID)
+	if plans, err := open.inventory.OpenPlans(ctx); err == nil {
+		if module := moduleAskedAs(plans, result.ID); module != "" {
+			planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID)
+			return
+		}
+	}
 	entries, err := open.inventory.Catalogued(ctx)
 	if err != nil {
 		return
 	}
 	for _, e := range entries {
 		if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
-			asked, _ := link.BuildAskedAt(result.ID)
-			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked)
+			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID)
 			return
 		}
 	}
 }
 
+// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing.
+func moduleAskedAs(plans []inventory.Plan, id string) string {
+	if id == "" {
+		return ""
+	}
+	for _, p := range plans {
+		if p.Tier >= len(p.Tiers) {
+			continue
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if s := p.Modules[m]; s != nil && s.Build == id {
+				return m
+			}
+		}
+	}
+	return ""
+}
+
 func repositoryMatches(a, b string) bool {
 	trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
 	return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
@@ -867,7 +939,7 @@ type planStatus struct {
 	Late       bool      `json:"late"`
 }
 
-func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
+func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
 	out := make([]planStatus, 0, len(plans))
 	for _, p := range plans {
 		ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
@@ -878,6 +950,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 				ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
 			}
 			ps.Late = now.Sub(p.Updated) > planWaitBound
+			// Paused is a person's decision, not lateness (novox/hq ADR 0219).
+			if waiting, paused := pausedWaiting(p, pause, now); paused {
+				ps.Waiting, ps.Late = waiting, false
+			}
 		}
 		out = append(out, ps)
 	}
@@ -885,12 +961,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 }
 
 // openPlans is the open plans among the recent ones, and how many have waited past the bound.
-func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
+func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
 	var open []inventory.Plan
 	late := 0
 	for _, p := range plans {
 		if p.Open() {
 			open = append(open, p)
+			if _, paused := pausedWaiting(p, pause, time.Now()); paused {
+				continue
+			}
 			if time.Since(p.Updated) > planWaitBound {
 				late++
 			}
@@ -923,7 +1002,7 @@ func plansCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
 		for i, tier := range p.Tiers {
 			marker := " "
 			if i == p.Tier && p.Open() {
@@ -938,6 +1017,9 @@ func plansCommand(ctx context.Context, args []string) error {
 					if s.Commit != "" {
 						state += " from " + short(s.Commit)
 					}
+					if s.Build != "" && s.State != "built" {
+						state += " (" + s.Build + ")"
+					}
 					if s.Why != "" {
 						state += ": " + s.Why
 					}
@@ -950,6 +1032,15 @@ func plansCommand(ctx context.Context, args []string) error {
 	if *whatIf != "" {
 		return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
 	}
+	// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
+	if len(positionals) == 2 && positionals[0] == "retry" {
+		said, err := retryPlan(ctx, open, positionals[1])
+		if err != nil {
+			return err
+		}
+		fmt.Println(said)
+		return nil
+	}
 	// `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one
 	// waiting on a report that cannot come — so it stops reading as work in progress. Marked failed
 	// with who ended it; what it asked still builds and registers.
@@ -991,8 +1082,9 @@ func plansCommand(ctx context.Context, args []string) error {
 		fmt.Println("no merge has produced a plan yet")
 		return nil
 	}
+	pause := buildSeatPause(ctx, inv, plans)
 	for _, p := range plans {
-		fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
 	}
 	return nil
 }
@@ -1094,7 +1186,12 @@ func splitList(s string) []string {
 // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
 // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
 // Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
-func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build) bool {
+//
+// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR
+// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is
+// found by nothing else.
+func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build,
+	byID map[string]inventory.Build) bool {
 	changed := false
 	for _, m := range tier {
 		s := p.Modules[m]
@@ -1103,6 +1200,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 		}
 		var outcome *inventory.Build
 		for i := range recorded[m] {
+			// Asked under an id, only that id's record answers (ADR 0219), and it is looked up below.
+			if s.Build != "" {
+				break
+			}
 			b := recorded[m][i]
 			if b.At.Before(*s.AskedAt) {
 				break
@@ -1114,6 +1215,9 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 			}
 			outcome = &b
 		}
+		if own, found := byID[s.Build]; s.Build != "" && found {
+			outcome = &own
+		}
 		if outcome == nil {
 			continue
 		}

cmd/mesh-controller/release_plan_test.go

@@ -145,7 +145,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 		// Only a build from before the ask: not this ask's outcome.
 		"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
 	}
-	if !settleFromRecords(&p, p.Tiers[0], records) {
+	if !settleFromRecords(&p, p.Tiers[0], records, nil) {
 		t.Fatal("nothing settled, though the controller's build is recorded after the ask")
 	}
 	if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
@@ -162,13 +162,13 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 	late := map[string][]inventory.Build{"postgres": {
 		{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
 	}}
-	if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" {
+	if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
 		t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
 	}
 	// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
 	late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
 		Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
-	if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" ||
+	if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
 		r.Modules["postgres"].Commit != "4bcd5f73" {
 		t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
 	}
@@ -176,7 +176,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
 	// A failure recorded after the ask fails the plan, as hearing it would have.
 	q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
 		Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
-	settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}})
+	settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
 	if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
 		t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
 	}

cmd/mesh-controller/seatverbs.go

@@ -103,10 +103,48 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		if id := str("close"); id != "" {
 			return []string{"plans", "close", id}, nil
 		}
+		if id := str("retry"); id != "" {
+			return []string{"plans", "retry", id}, nil
+		}
 		if id := str("id"); id != "" {
 			return []string{"plans", id}, nil
 		}
 		return []string{"plans"}, nil
+	// The build queue (novox/hq ADR 0219).
+	case "queue":
+		return []string{"queue"}, nil
+	case "cancel", "kill":
+		if err := need("id"); err != nil {
+			return nil, err
+		}
+		return []string{verb, str("id")}, nil
+	case "clear":
+		if str("dead") == "true" {
+			return []string{"clear", "--dead"}, nil
+		}
+		return []string{"clear"}, nil
+	case "rebuild":
+		if err := need("what"); err != nil {
+			return nil, err
+		}
+		return []string{"rebuild", str("what")}, nil
+	case "replay":
+		if err := need("id"); err != nil {
+			return nil, err
+		}
+		argv := []string{"replay", str("id")}
+		if str("register") == "true" {
+			argv = append(argv, "--register")
+		}
+		if str("older") == "true" {
+			argv = append(argv, "--older")
+		}
+		return argv, nil
+	case "pause", "resume":
+		if n := str("node"); n != "" {
+			return []string{verb, n}, nil
+		}
+		return []string{verb}, nil
 	case "plan":
 		if err := need("node"); err != nil {
 			return nil, err

cmd/mesh-controller/status.go

@@ -138,14 +138,14 @@ func printStatus(asked answers) error {
 			len(quiet), strings.Join(said, "\n  "))
 	}
 
-	if open, late := openPlans(asked.plans); len(open) > 0 {
+	if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
 		fmt.Printf("%d plan(s) open", len(open))
 		if late > 0 {
 			fmt.Printf(", %d waiting past %s", late, planWaitBound)
 		}
 		fmt.Println(":")
 		for _, p := range open {
-			fmt.Printf("  %s\n", planLine(p, time.Now()))
+			fmt.Printf("  %s\n", planLineWith(p, time.Now(), asked.paused))
 		}
 		fmt.Println()
 	}
@@ -420,6 +420,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	if err != nil {
 		return answers{}, err
 	}
+	// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
+	out.paused = buildSeatPause(ctx, inv, out.plans)
 
 	// And which machines are not running what the mesh would send them. The same question as a
 	// module being behind its source, one level down: that one says the catalogue is out of date,

internal/broker/cancelled.go

@@ -0,0 +1,92 @@
+package broker
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// A seat's cancelled set (novox/hq ADR 0219).
+//
+// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
+// race no ordering on one side closes: a holder may fetch the ask in the moment between the
+// controller reading the queue and deleting the message, and build what a person cancelled. So the
+// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
+// there after taking it and before building: listed, it terminates the ask and announces it failed
+// rather than starting it.
+//
+// **A key-value bucket, because that is the shape the bus already has for "a small set the
+// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
+// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
+// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
+// three objects of one work queue read as one family; asserted by the controller with the queue,
+// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
+
+// CancelledSetName is the bucket holding a seat's cancelled asks.
+func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
+
+// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
+// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
+func hasCancelledSet(seat string) bool {
+	for _, s := range seatsTheControllerAsks {
+		if s == seat {
+			return true
+		}
+	}
+	return false
+}
+
+// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
+// own, and never one to report as state nothing declares.
+func IsCancelledSet(bucket string) bool {
+	return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
+}
+
+// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
+const cancelledSetAge = 7 * 24 * time.Hour
+
+// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
+type CancelledSetAsserter interface {
+	EnsureCancelledSet(seat string) error
+}
+
+// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
+func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
+	for _, s := range seats {
+		if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
+			continue
+		}
+		if err := a.EnsureCancelledSet(s.Name); err != nil {
+			return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
+		}
+	}
+	return nil
+}
+
+// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
+// Direct reads on, which is how a holder looks an id up with one request.
+func (j *JetStream) EnsureCancelledSet(seat string) error {
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket: CancelledSetName(seat),
+		Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
+			"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
+			"so an ask fetched in that moment is ended rather than built", seat),
+		History:      1,
+		TTL:          cancelledSetAge,
+		MaxValueSize: 4 * 1024,
+		MaxBytes:     4 * 1024 * 1024,
+		Storage:      jetstream.FileStorage,
+	}); err != nil {
+		return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
+	}
+	return nil
+}

internal/broker/cancelled_test.go

@@ -0,0 +1,73 @@
+package broker
+
+import "testing"
+
+// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own
+// machine's subjects, and says whether it is paused under its own machine's name; the controller may
+// ask any machine's holder its verbs (novox/hq ADR 0219).
+func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) {
+	seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"},
+		Emits:  []string{"started", "built", "log.*", "paused.*"},
+		Serves: []string{"current", "kill", "pause", "resume"}}
+	holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent",
+		Holds: []Seat{seat}, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
+	hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
+	has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.ace")
+	hasNot(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*")
+	has(t, holder.Publish, "mesh.seat.node-build-agent.event.log.*")
+	has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace")
+	hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14")
+
+	// A module's own seat's queue is its own affair: no cancelled set, no grant for one.
+	other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram",
+		Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"})
+	hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>")
+
+	controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>")
+
+	if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" ||
+		!IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") {
+		t.Error("the cancelled set is not named as its seat's family")
+	}
+}
+
+// Only the work queues the controller asks get a cancelled set.
+func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) {
+	var asserted []string
+	a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil })
+	if err := RaiseCancelledSets(a, []DeclaredSeat{
+		{Name: "node-build-agent", Accepts: []string{"build"}},
+		{Name: "telegram-sender", Accepts: []string{"send"}},
+		{Name: "mesh-controller"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if len(asserted) != 1 || asserted[0] != "node-build-agent" {
+		t.Fatalf("asserted %v", asserted)
+	}
+}
+
+type asserterFunc func(string) error
+
+func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) }
+
+// A seat's cancelled set is never reported as state nothing declares.
+func TestACancelledSetIsNotUndeclaredState(t *testing.T) {
+	undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(undeclared) != 1 || undeclared[0] != "gone_state" {
+		t.Fatalf("undeclared %v", undeclared)
+	}
+}
+
+type fakeBuckets struct{ names []string }
+
+func (f fakeBuckets) EnsureBucket(Bucket) error      { return nil }
+func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil }

internal/broker/nats.go

@@ -124,6 +124,10 @@ type Principal struct {
 // goes with the retired seat row.
 var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
 
+// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
+// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
+var perMachineEvents = map[string]bool{"paused.*": true}
+
 // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
 // controller may answer an enrolment is derived from the same constant the user is named from.
 const enrolmentPrefix = "enrol"
@@ -221,6 +225,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// the role, and whichever machine holding it is idle takes it.
 		for _, seat := range seatsTheControllerAsks {
 			pub = append(pub, "mesh.seat."+seat+".accept.>")
+			// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
+			// building, kill it, pause it, resume it. The queue is the controller's to show and to
+			// change, and what one machine is doing with an ask it took only that machine can say.
+			pub = append(pub, "mesh.seat."+seat+".tool.>")
 		}
 		// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
 		// facts under the seat it holds, because a role's events belong to the role and keep their
@@ -404,10 +412,26 @@ func PermissionsFor(p Principal) (Permissions, error) {
 				"$JS.API.CONSUMER.INFO."+stream+"."+worker,
 				"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
 				"$JS.ACK."+stream+"."+worker+".>")
+			// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
+			// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
+			// moment the controller cancelled it ends it instead of building it. Read, never written:
+			// the set is the controller's, and only the work queues the controller asks — and so may
+			// cancel from — have one.
+			if hasCancelledSet(s.Name) {
+				set := CancelledSetName(s.Name)
+				pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>")
+			}
 			for _, a := range s.Accepts {
 				sub = append(sub, seatSubject(s, "accept", a))
 			}
 			for _, e := range s.Emits {
+				// **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped
+				// seat, an event about the holder itself carries the machine as its last token, and
+				// each holder is granted its own machine's alone.
+				if s.Scope == "node" && p.Node != "" && perMachineEvents[e] {
+					pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node))
+					continue
+				}
 				pub = append(pub, seatSubject(s, "event", e))
 			}
 			for _, t := range s.Serves {

internal/broker/state.go

@@ -160,7 +160,8 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
 		return nil, fmt.Errorf("listing the bus's state: %w", err)
 	}
 	for _, n := range names {
-		if !declared[n] {
+		// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
+		if !declared[n] && !IsCancelledSet(n) {
 			undeclared = append(undeclared, n)
 		}
 	}

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
         subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/builder/builder.go

@@ -761,6 +761,9 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err
 	tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " "))
 	cmd := exec.CommandContext(ctx, name, args...)
 	cmd.Dir = dir
+	// **Ended whole when the build is** (novox/hq ADR 0219): its own process group, killed as one
+	// when the context ends, so a build killed by hand leaves no `git` or `docker` child running.
+	inItsOwnGroup(cmd)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		tell("run", "! %s %s failed after %s", name, args[0], since(started))

internal/builder/kill.go

@@ -0,0 +1,78 @@
+package builder
+
+import (
+	"context"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// A build killed by hand (novox/hq ADR 0219).
+//
+// A build is a tree of commands — git, then docker, and docker's own children — and a container the
+// docker client started keeps running when the client dies. So ending one by hand is three things:
+// the build's context is cancelled, which kills each command's whole process group rather than the
+// one process the context knows; every container the build started carries the build's id as a
+// label, so what outlived its client is found and removed by that label; and the holder announces
+// the outcome itself, since nothing else will.
+
+// BuildLabel is the label every container a build starts carries, valued with the build's id.
+const BuildLabel = "mesh.build"
+
+// KillWait is how long a command killed with its build may take to let go of its output before it
+// is abandoned: a grandchild holding the pipe open must not hold the build open with it.
+const KillWait = 10 * time.Second
+
+// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its
+// context ends.
+func inItsOwnGroup(cmd *exec.Cmd) {
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	cmd.Cancel = func() error {
+		if cmd.Process == nil {
+			return nil
+		}
+		// The negative pid is the group: the command and everything it started.
+		if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
+			return cmd.Process.Kill()
+		}
+		return nil
+	}
+	cmd.WaitDelay = KillWait
+}
+
+// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
+// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
+// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
+func Labelled(run Runner, id string) Runner {
+	return func(ctx context.Context, dir string, name string, args ...string) (string, error) {
+		return run(ctx, dir, name, LabelledArgs(name, args, id)...)
+	}
+}
+
+// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`.
+func LabelledArgs(name string, args []string, id string) []string {
+	if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
+		return args
+	}
+	out := make([]string, 0, len(args)+2)
+	out = append(out, "run", "--label", BuildLabel+"="+id)
+	return append(out, args[1:]...)
+}
+
+// RemoveContainersOf removes every container labelled with the build's id, running or not, and
+// says how many. Run with a context of its own: the build's is the one that was just cancelled.
+func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) {
+	out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id)
+	if err != nil {
+		return 0, err
+	}
+	ids := strings.Fields(out)
+	if len(ids) == 0 {
+		return 0, nil
+	}
+	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+		return 0, err
+	}
+	return len(ids), nil
+}

internal/builder/kill_test.go

@@ -0,0 +1,103 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills
+// the command's whole process group, not the one process the context knows about.
+func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) {
+	dir := t.TempDir()
+	child := filepath.Join(dir, "child")
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	began := time.Now()
+	go func() {
+		// A shell that starts a grandchild and waits on it: killing the shell alone would leave the
+		// grandchild running, holding the output open.
+		_, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`)
+		done <- err
+	}()
+	var pid int
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" {
+			pid, _ = strconv.Atoi(strings.TrimSpace(string(raw)))
+			break
+		}
+	}
+	if pid == 0 {
+		t.Fatal("the command never started its child")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("a killed command reported success")
+		}
+	case <-time.After(KillWait + 5*time.Second):
+		t.Fatal("the killed command never returned")
+	}
+	if took := time.Since(began); took > KillWait {
+		t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took)
+	}
+	for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) {
+			return
+		}
+	}
+	_ = syscall.Kill(pid, syscall.SIGKILL)
+	t.Fatalf("the grandchild %d outlived the kill", pid)
+}
+
+// Every `docker run` a build starts carries its id as a label; nothing else is touched.
+func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) {
+	got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
+	if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("docker run became %v", got)
+	}
+	for _, c := range []struct {
+		name string
+		args []string
+	}{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} {
+		if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) {
+			t.Errorf("%s %v became %v", c.name, c.args, got)
+		}
+	}
+	var ran [][]string
+	run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, append([]string{name}, args...))
+		return "", nil
+	}, "build-2")
+	_, _ = run(context.Background(), "", "docker", "run", "img")
+	if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}
+
+// What a kill removes is found by the label, and only what it finds.
+func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
+	var ran []string
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\nc2\n", nil
+		}
+		return "", nil
+	}
+	n, err := RemoveContainersOf(context.Background(), run, "build-3")
+	if err != nil || n != 2 {
+		t.Fatalf("%d %v", n, err)
+	}
+	if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}

internal/catalogue/seats.go

@@ -117,8 +117,16 @@ var defaultSeats = append([]Seat{
 	// **Node-scoped, and every holder takes from one queue** (novox/hq ADR 0190): a build is asked of
 	// the role, and whichever machine holding the seat is idle pulls it. One holder per machine is
 	// what the scope says; sharing the work is what a seat's queue has always done.
+	//
+	// **And its holder answers for the build it is running** (novox/hq ADR 0219): what it is
+	// building, kill it, take nothing new, take again. The queue as a whole is the controller's to
+	// show and change (`queue`, `cancel`, `clear`); what one machine does with an ask it already
+	// took only that machine can do. `paused.<node>` is each holder saying whether it takes work, so
+	// the controller can tell a plan waiting on a paused seat from one that is late.
 	{Name: "node-build-agent", Scope: ScopeNode,
-		Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
+		Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*", "paused.*"},
+		Serves:   buildAgentVerbs(),
+		Decision: "novox/hq ADR 0190, ADR 0219"},
 	// **Retired by ADR 0190, kept while a manifest still claims it.** The one build machine's seat.
 	// A claim to a seat the mesh no longer defines is refused, and the module holding this one is
 	// assigned on a live machine until build-agent replaces it — removing the row first would make
@@ -581,3 +589,23 @@ func loginShellVerbs() []Verb {
 			}, []string{"command"})},
 	}
 }
+
+// buildAgentVerbs are what a holder of node-build-agent answers on its own machine (novox/hq ADR
+// 0219). One build at a time per holder (ADR 0190), so "the build running here" is one or none.
+func buildAgentVerbs() []Verb {
+	return []Verb{
+		{Name: "current", Description: "The build this machine is running — its id, repository, path, " +
+			"the step it is at, when it started and for how long — or none; and whether this machine is " +
+			"paused, taking no new build.",
+			Input: schema(map[string]string{}, nil)},
+		{Name: "kill", Description: "End the build with this id, running here: its commands and the " +
+			"containers it started are stopped, and its outcome is announced as failed, killed by hand — " +
+			"settled, so it is not handed to another machine.",
+			Input: schema(map[string]string{"id": "the build's id, as `queue` or `current` says it"}, []string{"id"})},
+		{Name: "pause", Description: "Take no new build on this machine until resumed; a build running " +
+			"here finishes. Kept across a restart of the holder.",
+			Input: schema(map[string]string{}, nil)},
+		{Name: "resume", Description: "Take builds again on this machine.",
+			Input: schema(map[string]string{}, nil)},
+	}
+}

internal/catalogue/verbs.go

@@ -97,6 +97,7 @@ var ControllerVerbs = []Verb{
 			"id":         "a plan's id (as `plans` lists them): that plan, tier by tier",
 			"stop":       "a plan's id: stop it — what was asked still builds, nothing further is asked",
 			"close":      "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
+			"retry":      "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)",
 			"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
 			"paths":      "with repository: the files the merge would change, comma-separated, from the repository's root",
 			"modules":    "with repository: or the modules it would change, comma-separated",
@@ -156,6 +157,36 @@ var ControllerVerbs = []Verb{
 		Input: schema(map[string]string{
 			"command": "the command line, as the controller's binary takes it; quotes group a word with spaces",
 		}, []string{"command"})},
+	// The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a
+	// failed outcome for it, so a plan waiting on it fails visibly instead of hanging.
+	{Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " +
+		"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked.",
+		Input: schema(nil, nil)},
+	{Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " +
+		"cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.",
+		Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
+	{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
+		"recorded failed, cancelled by hand. Never touches one in flight.",
+		Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)},
+	{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
+		"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
+		Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
+	{Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " +
+		"A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " +
+		"is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.",
+		Input: schema(map[string]string{
+			"id":       "the build's id",
+			"register": "\"true\" to register what it builds",
+			"older":    "\"true\", with register: even though a newer build of the module is registered",
+		}, []string{"id"})},
+	{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
+		"announces it failed, killed by hand — settled, never handed to another machine.",
+		Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
+	{Name: "pause", Description: "The build seat's holder on one machine — or every holder — takes no new build until resumed; " +
+		"a build running finishes. Kept across a restart of the holder. A plan waiting on a paused seat says so and is not late.",
+		Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
+	{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
+		Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
 	{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
 		"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
 		Input: schema(map[string]string{

internal/inventory/builds.go

@@ -3,8 +3,11 @@ package inventory
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"sort"
 	"time"
+
+	"github.com/jackc/pgx/v5"
 )
 
 // What has been built.
@@ -161,6 +164,33 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
 	return out, rows.Err()
 }
 
+// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
+// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
+// repository, path, ref and commit from. False when no outcome with that id was recorded.
+func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
+	var b Build
+	var made []byte
+	var asked *time.Time
+	err := i.store.Pool().QueryRow(ctx,
+		`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
+		        coalesce(source_path,''), asked, at
+		   from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
+		&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return Build{}, false, nil
+	}
+	if err != nil {
+		return Build{}, false, err
+	}
+	if asked != nil {
+		b.Asked = *asked
+	}
+	if err := json.Unmarshal(made, &b.Made); err != nil {
+		return Build{}, false, err
+	}
+	return b, true, nil
+}
+
 // Held is every artifact this mesh has built, keyed "<module>/<artifact>".
 //
 // **The successful build of each module asked last wins**, which is the same rule the rest of the

internal/inventory/plans.go

@@ -49,6 +49,11 @@ type PlanModule struct {
 	FirstAt *time.Time `json:"first_at,omitempty"`
 	Commit  string     `json:"commit,omitempty"`
 	Why     string     `json:"why,omitempty"`
+	// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
+	// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
+	// its module's name included. Empty in a plan from before it was kept, which is matched by
+	// module, or by repository and path, as before.
+	Build string `json:"build,omitempty"`
 }
 
 // The states a plan passes through.

internal/link/builds_nats.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -105,7 +106,20 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
 	waiting, cancelWait := context.WithTimeout(ctx, wait)
 	defer cancelWait()
 	for {
-		msg, err := outcomes.NextMsgWithContext(waiting)
+		// **A wait that hears a cancel** (novox/hq ADR 0219). A person cancelling an ask records its
+		// failure without publishing the role's outcome — that subject is the holders', and the
+		// controller may not speak for them — so the waiter also looks, every while, at the cancelled
+		// set the cancel wrote first. A kill needs no look: the holder announces it as any outcome.
+		slice, endSlice := context.WithTimeout(waiting, cancelLook)
+		msg, err := outcomes.NextMsgWithContext(slice)
+		endSlice()
+		if errors.Is(err, context.DeadlineExceeded) && waiting.Err() == nil {
+			if cancelled, _ := IsCancelled(b.js.Conn(), b.role(), request.ID); cancelled {
+				return BuildResult{ID: request.ID, Repository: request.Repository, Path: request.Path,
+					Ref: request.Ref, Source: request.Source, DryRun: request.DryRun, Failed: CancelledByHand}, nil
+			}
+			continue
+		}
 		switch {
 		case errors.Is(err, context.DeadlineExceeded):
 			return BuildResult{}, waitingFor(wait)
@@ -124,6 +138,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
 	}
 }
 
+// cancelLook is how often a waiting asker looks whether its ask was cancelled.
+var cancelLook = 2 * time.Second
+
 // --- the machine's side ---------------------------------------------------------------------
 
 type natsMachine struct {
@@ -131,6 +148,14 @@ type natsMachine struct {
 	on   string
 	seat string
 	sub  *nats.Subscription
+	opts MachineOptions
+}
+
+// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
+type MachineOptions struct {
+	// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
+	// already running finishes. Nil is never paused.
+	Paused func() bool
 }
 
 // MachineOverNATS takes build work from the current build role.
@@ -145,6 +170,18 @@ func MachineOverNATSOn(js *broker.JetStream, on, seat string) BuildMachine {
 	return &natsMachine{js: js, on: on, seat: seat}
 }
 
+// MachineOverNATSWith is MachineOverNATSOn for a holder that can be paused (novox/hq ADR 0219).
+func MachineOverNATSWith(js *broker.JetStream, on, seat string, opts MachineOptions) BuildMachine {
+	return &natsMachine{js: js, on: on, seat: seat, opts: opts}
+}
+
+// pausedPoll is how often a paused holder looks again whether it was resumed, and pausedFetch how
+// long one pull of a holder that can be paused waits.
+const (
+	pausedPoll  = 2 * time.Second
+	pausedFetch = 5 * time.Second
+)
+
 func (m *natsMachine) Close() {
 	if m.sub != nil {
 		_ = m.sub.Unsubscribe()
@@ -189,9 +226,27 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 		if ctx.Err() != nil {
 			return nil
 		}
+		// **Paused takes nothing new** (novox/hq ADR 0219). Asked before the fetch, never during a
+		// build: what this machine already took it finishes, and what it has not taken stays in the
+		// queue for another holder — or for this one, resumed.
+		if m.opts.Paused != nil && m.opts.Paused() {
+			select {
+			case <-ctx.Done():
+				return nil
+			case <-time.After(pausedPoll):
+			}
+			continue
+		}
 		// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
 		// machine with nothing to build, and is asked again.
-		fetched, err := sub.Fetch(1, nats.Context(ctx))
+		// Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull
+		// already waiting within that, rather than when the client's own wait runs out.
+		asking, endAsking := ctx, func() {}
+		if m.opts.Paused != nil {
+			asking, endAsking = context.WithTimeout(ctx, pausedFetch)
+		}
+		fetched, err := sub.Fetch(1, nats.Context(asking))
+		endAsking()
 		switch {
 		case ctx.Err() != nil:
 			// Ours ended: the machine is being stopped.
@@ -213,6 +268,13 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			return fmt.Errorf("the bus stopped delivering build work: %w", err)
 		}
 		for _, msg := range fetched {
+			// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
+			// the one that arrived in that moment. Handed back at once for another holder — counted as
+			// a delivery, which a pause landing exactly then costs and nothing else does.
+			if m.opts.Paused != nil && m.opts.Paused() {
+				_ = msg.Nak()
+				continue
+			}
 			var request BuildRequest
 			if err := json.Unmarshal(msg.Data, &request); err != nil {
 				// Unreadable: terminated rather than retried, because the next attempt reads the same
@@ -220,12 +282,25 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 				_ = msg.Term()
 				continue
 			}
+			// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
+			// id into the seat's cancelled set before deleting the ask, so one taken in between is
+			// ended here, terminated rather than redelivered, and its outcome said as failed — never
+			// built. A set that cannot be read is said and the ask built: a cancel is a person's
+			// exception, and a holder that refused every build while its grant was missing would
+			// stop the mesh building for it.
+			build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
+			if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
+				fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
+			} else if cancelled {
+				endCancelled(ctx, build)
+				continue
+			}
 			// A build outlives the acknowledgement window many times over; said while it runs,
 			// as the controller says it for its own long handlers, so the server neither hands
 			// the ask to a second machine nor counts the wait against its deliveries.
 			working := make(chan struct{})
 			go stillWorking(msg, working)
-			do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat})
+			do(ctx, build)
 			close(working)
 		}
 	}
@@ -307,3 +382,17 @@ func (b *natsBuild) Say(step, message string) {
 }
 
 func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) }
+
+// endCancelled settles an ask that was cancelled as it was taken: its outcome said as failed, as the
+// controller already recorded it, so anybody still waiting on it hears the answer — then
+// terminated, so the queue neither keeps nor redelivers it.
+func endCancelled(ctx context.Context, b *natsBuild) {
+	r := b.request
+	fmt.Fprintf(os.Stderr, "%s was cancelled by hand as this machine took it; not built\n", r.ID)
+	result := BuildResult{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: b.on,
+		Source: r.Source, DryRun: r.DryRun, Failed: CancelledByHand}
+	if err := b.Announce(ctx, result); err != nil {
+		fmt.Fprintf(os.Stderr, "cannot say %s was cancelled: %v\n", r.ID, err)
+	}
+	_ = b.msg.Term()
+}

internal/link/builds_nats_test.go

@@ -303,8 +303,9 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi
 	case <-time.After(2 * time.Second):
 	}
 	// One finishes, and only then is the third taken — by that machine, the one that is free.
-	close(release["anchor"])
-	release["anchor"] = make(chan struct{})
+	// Released by a send, never by replacing the channel: the map is read by both machines' builds
+	// while this runs, and writing it raced them.
+	release["anchor"] <- struct{}{}
 	select {
 	case got := <-took:
 		if got.machine != "anchor" {
@@ -318,7 +319,7 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi
 	// an explicit hand-back, because the real wait is a minute. Here: the laptop goes, anchor
 	// finishes, and with nothing queued nothing more is taken by the machine that is left.
 	machines["laptop"].Close()
-	close(release["anchor"])
+	release["anchor"] <- struct{}{}
 	select {
 	case got := <-took:
 		t.Fatalf("%s took %s; the queue should be empty", got.machine, got.id)

internal/link/queue.go

@@ -0,0 +1,465 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// The build queue, read and changed by hand (novox/hq ADR 0219).
+//
+// **A queue nobody can see is a queue nobody can trust.** Until this, the build seat's work queue
+// was visible only as its consequences: a plan LATE with nothing to say why, a build that never
+// started because five asks ahead of it were waiting on a laptop that was shut. ADR 0219 makes the
+// queue the controller's to show and change, and the build running on one machine that machine's
+// holder's to end.
+//
+// Three kinds of ask are in the seat's stream at any moment, told apart by the worker consumer
+// every holder pulls from:
+//
+//   - **waiting** — after the last one the worker handed out (stream seq > delivered.stream_seq);
+//   - **in flight** — handed out and not yet settled, which the holder that took it said with its
+//     `started` event, and which the worker counts among its acks pending;
+//   - **dead** — handed out as many times as the worker allows and never settled. A work queue
+//     drops what it settles, so one still in the stream behind the worker is either in flight or
+//     dead; the started events and the worker's pending count say which.
+//
+// Everything that drops an ask leaves a failed outcome for it, recorded the way a failed build's is
+// (`cancelled by hand`, `killed by hand`), so a plan waiting on it fails visibly rather than waiting
+// for ever on an ask nobody will answer.
+
+// The words an outcome says when a person ended the ask.
+const (
+	CancelledByHand = "cancelled by hand"
+	KilledByHand    = "killed by hand"
+)
+
+// Ask states in a queue.
+const (
+	AskWaiting  = "waiting"
+	AskInFlight = "in flight"
+	AskDead     = "dead"
+)
+
+// QueuedAsk is one ask in the seat's work queue.
+type QueuedAsk struct {
+	Seq     uint64       `json:"seq"`
+	Request BuildRequest `json:"-"`
+	ID      string       `json:"id"`
+	// Repository, Path and Ref are the ask's, as the request carried them; Held never travels out of
+	// here — it is every artifact the mesh has built, and a queue listing is not where that belongs.
+	Repository string    `json:"repository"`
+	Path       string    `json:"path,omitempty"`
+	Ref        string    `json:"ref,omitempty"`
+	AskedAt    time.Time `json:"asked-at,omitempty"`
+	State      string    `json:"state"`
+	// On and Started are the holder building an in-flight ask and when it started, from its started
+	// event. Was is the holder that started an in-flight ask and is no longer building it — handed
+	// back, to be handed out again — with Started its start there.
+	On      string    `json:"on,omitempty"`
+	Was     string    `json:"was-on,omitempty"`
+	Started time.Time `json:"started,omitempty"`
+}
+
+// Queue is the seat's work queue as it stands.
+type Queue struct {
+	Seat string `json:"seat"`
+	// Delivered is the last stream sequence the worker handed out; AckPending how many it handed out
+	// and has not had settled; MaxDeliver how many times it hands one ask out.
+	Delivered  uint64      `json:"delivered"`
+	AckPending int         `json:"ack-pending"`
+	MaxDeliver int         `json:"max-deliver"`
+	Asks       []QueuedAsk `json:"asks"`
+}
+
+// Of is the asks in one state, in queue order.
+func (q Queue) Of(state string) []QueuedAsk {
+	var out []QueuedAsk
+	for _, a := range q.Asks {
+		if a.State == state {
+			out = append(out, a)
+		}
+	}
+	return out
+}
+
+// Find is the ask with this id.
+func (q Queue) Find(id string) (QueuedAsk, bool) {
+	for _, a := range q.Asks {
+		if a.ID == id {
+			return a, true
+		}
+	}
+	return QueuedAsk{}, false
+}
+
+// BuildHeard is what the seat's events say about builds: the latest start of each id, and every id
+// whose outcome was heard.
+type BuildHeard struct {
+	Started  map[string]BuildStart
+	Outcomes map[string]bool
+}
+
+// ClassifyQueue says which state each ask is in. Pure: the stream's asks in sequence order, what the
+// worker says, and what the seat's events said.
+//
+// **In flight is what the worker counts handed out and unsettled**, at most AckPending of the asks
+// behind it, given out in this order:
+//
+//  1. what a machine is building now — its latest start, no outcome heard (a holder builds one ask
+//     at a time, ADR 0190), newest start first;
+//  2. what a machine started and is no longer building — a holder restarted mid-build, the ask
+//     handed back and waiting to be handed out again while it has deliveries left — newest start
+//     first, naming the machine it was last on;
+//  3. what was taken and not yet said started.
+//
+// Only what is left after that is dead: so nothing behind the worker is dead while there are no more
+// of them than the worker counts pending. A machine that died mid-build keeps a latest start for
+// ever; the worker's count is what says the ask was handed out as often as it may be.
+//
+// **The worker's count is the server's, and it lags in one case**: an ask handed out its last time
+// and handed back is still counted pending until some holder pulls again, when the server finds it
+// past its deliveries and lets it go. Holders pull whenever they are idle, so this is a moment —
+// except while every holder is paused, when such an ask reads as in flight with no machine named.
+func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard BuildHeard) []QueuedAsk {
+	// Each machine's latest start.
+	latest := map[string]BuildStart{}
+	for _, s := range heard.Started {
+		at := startedAt(s)
+		if was, ok := latest[s.On]; !ok || at.After(startedAt(was)) {
+			latest[s.On] = s
+		}
+	}
+	current := map[string]BuildStart{}
+	for _, s := range latest {
+		if !heard.Outcomes[s.ID] {
+			current[s.ID] = s
+		}
+	}
+
+	out := make([]QueuedAsk, len(asks))
+	copy(out, asks)
+	var running, handedBack, unsaid []int
+	for i := range out {
+		a := &out[i]
+		if a.Seq > delivered {
+			a.State = AskWaiting
+			continue
+		}
+		a.State = AskDead
+		if s, ok := current[a.ID]; ok {
+			a.On, a.Started = s.On, startedAt(s)
+			running = append(running, i)
+			continue
+		}
+		if s, ever := heard.Started[a.ID]; ever {
+			a.Was, a.Started = s.On, startedAt(s)
+			handedBack = append(handedBack, i)
+			continue
+		}
+		unsaid = append(unsaid, i)
+	}
+	newestFirst := func(ix []int) {
+		sort.SliceStable(ix, func(x, y int) bool { return out[ix[x]].Started.After(out[ix[y]].Started) })
+	}
+	newestFirst(running)
+	newestFirst(handedBack)
+	slots := ackPending
+	for _, group := range [][]int{running, handedBack, unsaid} {
+		for _, i := range group {
+			if slots == 0 {
+				// Past what the worker counts: handed out as often as it may be.
+				out[i].On, out[i].Started = "", time.Time{}
+				continue
+			}
+			out[i].State = AskInFlight
+			slots--
+		}
+	}
+	return out
+}
+
+func startedAt(s BuildStart) time.Time {
+	t, _ := time.Parse(time.RFC3339Nano, s.At)
+	return t
+}
+
+// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the
+// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO,
+// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches.
+func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	q := Queue{Seat: seat}
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return q, err
+	}
+	stream, err := api.Stream(ctx, worker.Stream)
+	if err != nil {
+		return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
+	}
+	consumer, err := stream.Consumer(ctx, worker.Name)
+	switch {
+	case errors.Is(err, jetstream.ErrConsumerNotFound):
+		// No holder has ever been given a worker: everything in the stream is waiting.
+	case err != nil:
+		return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
+	default:
+		info, err := consumer.Info(ctx)
+		if err != nil {
+			return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
+		}
+		q.Delivered = info.Delivered.Stream
+		q.AckPending = info.NumAckPending
+		q.MaxDeliver = info.Config.MaxDeliver
+	}
+
+	// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
+	// is a handful of reads, never the history.
+	var asks []QueuedAsk
+	var seq uint64 = 1
+	for n := 0; n < 10000; n++ {
+		msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
+		if errors.Is(err, jetstream.ErrMsgNotFound) {
+			break
+		}
+		if err != nil {
+			return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
+		}
+		ask := QueuedAsk{Seq: msg.Sequence}
+		if json.Unmarshal(msg.Data, &ask.Request) == nil {
+			r := ask.Request
+			ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
+			if at, ok := BuildAskedAt(r.ID); ok {
+				ask.AskedAt = at
+			}
+		}
+		asks = append(asks, ask)
+		seq = msg.Sequence + 1
+	}
+
+	// What the events say, from shortly before the oldest ask behind the worker: its start, if any,
+	// came after it was asked.
+	var since time.Time
+	for _, a := range asks {
+		if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) {
+			since = a.AskedAt
+		}
+	}
+	heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
+	if len(asks) > 0 && q.Delivered > 0 {
+		if since.IsZero() {
+			since = time.Now().Add(-7 * 24 * time.Hour)
+		}
+		if heard, err = ReadBuildEvents(ctx, js, seat, since.Add(-time.Minute)); err != nil {
+			return q, err
+		}
+	}
+	q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard)
+	return q, nil
+}
+
+// ReadBuildEvents is every start and outcome the seat announced since a moment, from the events
+// stream, with a consumer of its own that is gone when this returns.
+func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) {
+	heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
+	// One token after `event`: started and built, never a build's log lines, which are two.
+	subject := "mesh.seat." + seat + ".event.*"
+	sub, err := js.Context().PullSubscribe(subject, "",
+		nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone())
+	if err != nil {
+		return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	for {
+		batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second))
+		if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
+			return heard, fmt.Errorf("reading %s: %w", subject, err)
+		}
+		for _, msg := range batch {
+			switch msg.Subject {
+			case BuildStartedOf(seat):
+				var s BuildStart
+				if json.Unmarshal(msg.Data, &s) == nil && s.ID != "" {
+					if was, ok := heard.Started[s.ID]; !ok || startedAt(s).After(startedAt(was)) {
+						heard.Started[s.ID] = s
+					}
+				}
+			case BuildOutcomeOf(seat):
+				var r BuildResult
+				if json.Unmarshal(msg.Data, &r) == nil && r.ID != "" {
+					heard.Outcomes[r.ID] = true
+				}
+			}
+		}
+		if len(batch) < 500 {
+			break
+		}
+		if ctx.Err() != nil {
+			return heard, ctx.Err()
+		}
+	}
+	return heard, nil
+}
+
+// --- the cancelled set ----------------------------------------------------------------------
+
+// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
+var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
+
+// Cancelled is what the controller writes for an ask it cancelled.
+type Cancelled struct {
+	At  string `json:"at"`
+	Why string `json:"why"`
+}
+
+// MarkCancelled puts an ask's id into the seat's cancelled set (novox/hq ADR 0219). The controller's
+// act, before it deletes the ask from the queue.
+func MarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error {
+	if !safeKey.MatchString(id) {
+		return fmt.Errorf("%q is not a build id", id)
+	}
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat))
+	if err != nil {
+		return fmt.Errorf("the cancelled set of %s is not on the bus — the controller asserts it at its "+
+			"start, so one older than this has not: %w", seat, err)
+	}
+	body, err := json.Marshal(Cancelled{At: time.Now().UTC().Format(time.RFC3339Nano), Why: CancelledByHand})
+	if err != nil {
+		return err
+	}
+	_, err = kv.Put(ctx, id, body)
+	return err
+}
+
+// UnmarkCancelled takes an ask's id out of the cancelled set: a cancel withdrawn, because the ask
+// was taken as it was being cancelled.
+func UnmarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error {
+	if !safeKey.MatchString(id) {
+		return nil
+	}
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat))
+	if err != nil {
+		return err
+	}
+	return kv.Delete(ctx, id)
+}
+
+// IsCancelled asks the seat's cancelled set whether an ask was cancelled: one direct read of one key,
+// which is all a holder is granted of it.
+func IsCancelled(conn *nats.Conn, seat, id string) (bool, error) {
+	if !safeKey.MatchString(id) {
+		// Not a key the controller could have written.
+		return false, nil
+	}
+	set := broker.CancelledSetName(seat)
+	reply, err := conn.Request("$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+"."+id, nil, 3*time.Second)
+	if err != nil {
+		return false, err
+	}
+	return cancelledFrom(reply)
+}
+
+// cancelledFrom reads a direct get's answer: a value is a cancel; not found, or a deletion marker,
+// is not.
+func cancelledFrom(reply *nats.Msg) (bool, error) {
+	if reply.Header != nil {
+		switch reply.Header.Get("Status") {
+		case "":
+		case "404":
+			return false, nil
+		default:
+			return false, fmt.Errorf("the cancelled set answered %s %s",
+				reply.Header.Get("Status"), reply.Header.Get("Description"))
+		}
+		if op := reply.Header.Get("KV-Operation"); op == "DEL" || op == "PURGE" {
+			return false, nil
+		}
+	}
+	return len(reply.Data) > 0, nil
+}
+
+// --- a holder's verbs -----------------------------------------------------------------------
+
+// NodeSeatToolSubject is where a node-scoped seat's verb is asked of one machine's holder (design 33
+// §4): the seat's verb subject with the machine as its last token.
+func NodeSeatToolSubject(seat, verb, node string) string {
+	return SeatToolSubject(seat, verb) + "." + node
+}
+
+// AskSeatTool asks one machine's holder of a node seat one of its verbs and reads its answer.
+func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string, args any,
+	timeout time.Duration) (Answer, error) {
+	body, err := json.Marshal(args)
+	if err != nil {
+		return Answer{}, err
+	}
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
+	switch {
+	case errors.Is(err, nats.ErrNoResponders):
+		return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
+			"older than the verb", node, seat, verb)
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
+		return Answer{}, fmt.Errorf("%s did not answer %s.%s within %s", node, seat, verb, timeout)
+	case err != nil:
+		return Answer{}, err
+	}
+	var answer Answer
+	if err := json.Unmarshal(reply.Data, &answer); err != nil {
+		return Answer{}, fmt.Errorf("%s answered %s.%s with something unreadable: %w", node, seat, verb, err)
+	}
+	return answer, nil
+}
+
+// --- a holder saying whether it takes work ----------------------------------------------------
+
+// HolderState is what a holder says about itself whenever it is paused or resumed, at its start,
+// and while it stays paused: whether it takes work (novox/hq ADR 0219). Retained on the events
+// stream under the machine's own subject, so the last one is the machine's state.
+type HolderState struct {
+	On     string `json:"on"`
+	Paused bool   `json:"paused"`
+	At     string `json:"at"`
+}
+
+// BuildPausedOf is where one machine's holder of a build seat says whether it takes work.
+func BuildPausedOf(seat, node string) string { return "mesh.seat." + seat + ".event.paused." + node }
+
+// PausedSaid reads what each machine last said about taking work. A machine that never said is not
+// in the answer — a holder older than ADR 0219 takes work and never pauses.
+func PausedSaid(js *broker.JetStream, seat string, nodes []string) (map[string]HolderState, error) {
+	out := map[string]HolderState{}
+	for _, node := range nodes {
+		msg, err := js.Context().GetLastMsg(broker.EventsStream, BuildPausedOf(seat, node))
+		if errors.Is(err, nats.ErrMsgNotFound) {
+			continue
+		}
+		if err != nil {
+			return out, err
+		}
+		var s HolderState
+		if json.Unmarshal(msg.Data, &s) == nil {
+			out[node] = s
+		}
+	}
+	return out, nil
+}

internal/link/queue_test.go

@@ -0,0 +1,336 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// The build queue read and changed by hand (novox/hq ADR 0219).
+
+// Which ask is waiting, in flight and dead, from the stream, the worker and the seat's events.
+func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) {
+	at := func(m int) string {
+		return time.Date(2026, 10, 5, 12, m, 0, 0, time.UTC).Format(time.RFC3339Nano)
+	}
+	asks := []QueuedAsk{
+		{Seq: 1, ID: "dead-1"},    // handed out five times, its machine moved on: dead
+		{Seq: 2, ID: "running-g"}, // g14's latest start, no outcome: in flight
+		{Seq: 3, ID: "running-a"}, // ace's latest start, no outcome: in flight
+		{Seq: 4, ID: "unsaid"},    // taken, not yet said: in flight while the worker counts it
+		{Seq: 5, ID: "waiting-1"}, // after the worker's last delivery
+		{Seq: 6, ID: "waiting-2"},
+	}
+	heard := BuildHeard{
+		Started: map[string]BuildStart{
+			"dead-1":    {ID: "dead-1", On: "g14", At: at(1)},
+			"running-g": {ID: "running-g", On: "g14", At: at(5)},
+			"running-a": {ID: "running-a", On: "ace", At: at(6)},
+			"done":      {ID: "done", On: "novox", At: at(7)},
+		},
+		Outcomes: map[string]bool{"done": true},
+	}
+	got := ClassifyQueue(asks, 4, 2, heard)
+	want := map[string]string{"dead-1": AskDead, "running-g": AskInFlight, "running-a": AskInFlight,
+		"unsaid": AskDead, "waiting-1": AskWaiting, "waiting-2": AskWaiting}
+	for _, a := range got {
+		if a.State != want[a.ID] {
+			t.Errorf("%s is %s, want %s", a.ID, a.State, want[a.ID])
+		}
+	}
+	q := Queue{Asks: got}
+	if a, _ := q.Find("running-a"); a.On != "ace" || a.Started.IsZero() {
+		t.Errorf("an ask in flight does not say where: %+v", a)
+	}
+
+	// **The worker's count bounds it**: with one pending, the machine whose start is oldest died
+	// with its ask.
+	got = ClassifyQueue(asks, 4, 1, heard)
+	q = Queue{Asks: got}
+	if a, _ := q.Find("running-a"); a.State != AskInFlight {
+		t.Errorf("running-a is %s", a.State)
+	}
+	if a, _ := q.Find("running-g"); a.State != AskDead || a.On != "" {
+		t.Errorf("past the worker's count running-g is %s on %q", a.State, a.On)
+	}
+	// **Handed back after a holder restarted mid-build**: started on g14, g14 then started something
+	// else, and the worker still counts it — it waits to be handed out again, and is not dead. With
+	// no more asks behind the worker than it counts pending, none is dead.
+	restarted := []QueuedAsk{{Seq: 1, ID: "dead-1"}, {Seq: 2, ID: "running-g"}}
+	for _, a := range ClassifyQueue(restarted, 2, 2, heard) {
+		if a.State != AskInFlight {
+			t.Errorf("%s is %s with two behind the worker and two pending", a.ID, a.State)
+		}
+		if a.ID == "dead-1" && (a.On != "" || a.Was != "g14") {
+			t.Errorf("an ask handed back reads on %q, was on %q", a.On, a.Was)
+		}
+	}
+	// The handed-back one takes a leftover slot before an ask nobody said started.
+	got = ClassifyQueue(asks, 4, 4, heard)
+	q = Queue{Asks: got}
+	for _, id := range []string{"dead-1", "running-g", "running-a", "unsaid"} {
+		if a, _ := q.Find(id); a.State != AskInFlight {
+			t.Errorf("with four pending %s is %s", id, a.State)
+		}
+	}
+	got = ClassifyQueue(asks, 4, 3, heard)
+	q = Queue{Asks: got}
+	if a, _ := q.Find("dead-1"); a.State != AskInFlight {
+		t.Errorf("the handed-back ask lost its slot to one nobody said: %s", a.State)
+	}
+	if a, _ := q.Find("unsaid"); a.State != AskDead {
+		t.Errorf("unsaid is %s", a.State)
+	}
+
+	// None pending: everything behind the worker is dead, and names no machine.
+	for _, a := range ClassifyQueue(asks, 4, 0, heard) {
+		if a.Seq <= 4 && (a.State != AskDead || a.On != "") {
+			t.Errorf("%s with nothing pending is %s on %q", a.ID, a.State, a.On)
+		}
+	}
+}
+
+// What a direct read of the cancelled set answers.
+func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) {
+	found := &nats.Msg{Header: nats.Header{"Nats-Subject": []string{"$KV.x.build-1"}}, Data: []byte(`{"why":"cancelled by hand"}`)}
+	if c, err := cancelledFrom(found); err != nil || !c {
+		t.Errorf("a value read as %v %v", c, err)
+	}
+	missing := &nats.Msg{Header: nats.Header{"Status": []string{"404"}}}
+	if c, err := cancelledFrom(missing); err != nil || c {
+		t.Errorf("not found read as %v %v", c, err)
+	}
+	deleted := &nats.Msg{Header: nats.Header{"KV-Operation": []string{"DEL"}}}
+	if c, err := cancelledFrom(deleted); err != nil || c {
+		t.Errorf("a deletion marker read as %v %v", c, err)
+	}
+	broken := &nats.Msg{Header: nats.Header{"Status": []string{"408"}, "Description": []string{"Request Timeout"}}}
+	if _, err := cancelledFrom(broken); err == nil {
+		t.Error("an error answer read as an answer")
+	}
+	if c, err := IsCancelled(nil, TheBuildMachine, "a.b.>"); err != nil || c {
+		t.Error("a pattern was looked up as a key")
+	}
+}
+
+// --- against a real server ----------------------------------------------------------------------
+
+func aBusWithACancelledSet(t *testing.T) *broker.JetStream {
+	t.Helper()
+	js := aBusWithTheBuildRole(t)
+	seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}
+	if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) })
+	return js
+}
+
+// **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by
+// the holder that took it — terminated, never built, its outcome said as failed.
+func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) {
+	js := aBusWithACancelledSet(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+
+	if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil {
+		t.Fatal(err)
+	}
+	outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = outcomes.Unsubscribe() }()
+	_ = js.Conn().Flush()
+	for _, id := range []string{"build-cancelled", "build-kept"} {
+		body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
+		if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	built := make(chan string, 2)
+	machine := MachineOverNATS(js, "anchor")
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			built <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	select {
+	case id := <-built:
+		if id != "build-kept" {
+			t.Fatalf("%s was built", id)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("the ask that was not cancelled was never built")
+	}
+	msg, err := outcomes.NextMsg(5 * time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var said BuildResult
+	if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" ||
+		said.Failed != CancelledByHand || said.On != "anchor" {
+		t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("the cancelled ask is still queued: terminated, it would not be")
+}
+
+// A paused holder takes nothing; resumed, it takes what waited.
+func TestNatsAPausedHolderTakesNothingNew(t *testing.T) {
+	js := aBusWithACancelledSet(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	paused := make(chan bool, 1)
+	paused <- true
+	isPaused := func() bool {
+		p := <-paused
+		paused <- p
+		return p
+	}
+	took := make(chan string, 1)
+	machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused})
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"})
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		t.Fatalf("a paused holder took %s", id)
+	case <-time.After(3 * time.Second):
+	}
+	q, err := ReadQueue(ctx, js, TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting {
+		t.Fatalf("while paused the queue reads %+v", q.Asks)
+	}
+	<-paused
+	paused <- false
+	select {
+	case id := <-took:
+		if id != "build-waits" {
+			t.Fatalf("took %s", id)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("resumed, the holder never took what waited")
+	}
+}
+
+// What a holder last said about taking work is read back per machine.
+func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} {
+		body, _ := json.Marshal(s)
+		if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 {
+		t.Fatalf("read back %+v", said)
+	}
+}
+
+// A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled
+// set, and a kill from the outcome the holder announces (novox/hq ADR 0219).
+func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) {
+	js := aBusWithACancelledSet(t)
+	was := cancelLook
+	cancelLook = 200 * time.Millisecond
+	t.Cleanup(func() { cancelLook = was })
+	ask := &natsBuilds{js: js, seat: TheBuildMachine}
+
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		_ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled")
+	}()
+	result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second)
+	if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" {
+		t.Fatalf("the waiter heard %+v (%v)", result, err)
+	}
+
+	// Killed: the holder announces the failure as any outcome, and the waiter has it.
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	machine := MachineOverNATS(js, "ace")
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			r := work.Request()
+			_ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand})
+			_ = work.Done()
+		})
+	}()
+	result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second)
+	if err != nil || result.Failed != KilledByHand || result.On != "ace" {
+		t.Fatalf("the waiter heard %+v (%v)", result, err)
+	}
+}
+
+// Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219).
+func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) {
+	js := aBusWithACancelledSet(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	// Not paused when it asks; paused by the time the answer is read.
+	var looks int
+	var mu sync.Mutex
+	paused := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		looks++
+		return looks > 1
+	}
+	took := make(chan string, 1)
+	machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused})
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID })
+	}()
+	time.Sleep(200 * time.Millisecond)
+	body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"})
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		t.Fatalf("a holder paused as its pull was answered built %s", id)
+	case <-time.After(2 * time.Second):
+	}
+	q, err := ReadQueue(ctx, js, TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" {
+		t.Fatalf("the ask is not back in the queue: %+v", q.Asks)
+	}
+}

internal/link/seattools.go

@@ -42,6 +42,18 @@ const RebindAfter = 30 * time.Second
 // tried again (2026-09-30). A refused subscription is therefore retried until it holds: the server
 // says so asynchronously and invalidates the subscription, which is what is checked.
 func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) {
+	return b.serveTools(seat, func(verb string) string { return SeatToolSubject(seat, verb) }, handlers, logger)
+}
+
+// ServeNodeSeatTools is ServeSeatTools for one machine's holder of a node-scoped seat (novox/hq ADR
+// 0159, ADR 0219): each verb on the seat's subject for this machine and no other, so a call names
+// the machine it is for and only that machine's holder answers it.
+func (b OverNATS) ServeNodeSeatTools(seat, node string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) {
+	return b.serveTools(seat, func(verb string) string { return NodeSeatToolSubject(seat, verb, node) }, handlers, logger)
+}
+
+func (b OverNATS) serveTools(seat string, subjectOf func(string) string, handlers map[string]ToolHandler,
+	logger *log.Logger) (func(), error) {
 	var subs []*nats.Subscription
 	done := make(chan struct{})
 	stop := func() {
@@ -52,7 +64,7 @@ func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, l
 	}
 	for verb, handle := range handlers {
 		verb, handle := verb, handle
-		subject := SeatToolSubject(seat, verb)
+		subject := subjectOf(verb)
 		bind := func() (*nats.Subscription, error) {
 			return b.Conn.QueueSubscribe(subject, "seat."+seat, func(msg *nats.Msg) {
 				// Its own goroutine per call: a slow `push` must not hold up a `status` asked beside it,