Keep a replay from moving the mesh backwards, and tighten the queue's edges (review of hq ADR 0219)

A registered replay asked now outranked newer asks of its module, and a plan
took any later outcome as its answer. A replay is now refused while the module
is asked anywhere, a plan module asked under an id is answered by that id
alone, and a rebuild of a commit asks what the module follows. An ask handed
back after a restart no longer reads as dead; a cancel that meets a start is
withdrawn; pause holds for an ask fetched as it lands; kill removes containers
before and after the build ends and says whether its outcome went out; a
holder may say only its own machine is paused.

cmd/mesh-builder/holder.go

@@ -54,7 +54,12 @@ type running struct {
 	started time.Time
 	cancel  context.CancelFunc
 	killed  bool
-	done    chan struct{}
+	// returned is the build's own work having ended, before a kill or not; outcome is what was then
+	// announced, and announced whether it went out.
+	returned  bool
+	outcome   string
+	announced bool
+	done      chan struct{}
 }
 
 // pausedFile is where the flag lives in the workspace.
@@ -152,13 +157,6 @@ func (h *holder) stepped(r *running, step string) {
 	h.mu.Unlock()
 }
 
-// wasKilled is whether a person killed this build.
-func (h *holder) wasKilled(r *running) bool {
-	h.mu.Lock()
-	defer h.mu.Unlock()
-	return r.killed
-}
-
 // currentBuild is what `current` answers.
 type currentBuild struct {
 	On      string `json:"on"`
@@ -204,10 +202,19 @@ func (h *holder) current() currentBuild {
 	return out
 }
 
-// kill ends the build with this id, if it is the one running here: its context cancelled — which
-// kills each command's process group — and the containers it started removed by their label. The
-// build's own goroutine then announces it failed, killed by hand, and settles the ask, so it is not
-// redelivered; this waits a while for that, to say it happened.
+// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
+// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
+// (killAnswer in the controller's queue.go, 75s).
+var (
+	killRemoves = 15 * time.Second
+	killWaits   = 20 * time.Second
+)
+
+// kill ends the build with this id, if it is the one running here and still working: its context
+// cancelled — which kills each command's process group — and the containers it started removed by
+// their label, once at once and again after the build has ended, so one created while it was being
+// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
+// ask so it is not redelivered; the answer says whether that happened.
 func (h *holder) kill(id string) (string, error) {
 	h.mu.Lock()
 	r := h.running
@@ -219,27 +226,68 @@ func (h *holder) kill(id string) (string, error) {
 		h.mu.Unlock()
 		return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
 	}
+	if r.returned {
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
+	}
 	r.killed = true
 	cancel, done := r.cancel, r.done
 	h.mu.Unlock()
 
 	cancel()
-	cleanup, stop := context.WithTimeout(context.Background(), 30*time.Second)
-	defer stop()
-	removed, err := builder.RemoveContainersOf(cleanup, h.remove, id)
-	containers := fmt.Sprintf("%d container(s) it started removed", removed)
-	if err != nil {
-		containers = "its containers could not be listed or removed: " + err.Error()
-	}
+	removed, removeErr := h.removeContainers(id)
+	ended := false
 	select {
 	case <-done:
-		return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as "+
-			"failed, %s — settled, so it is not handed to another machine", id, r.request.Repository, h.on,
-			containers, link.KilledByHand), nil
-	case <-time.After(30 * time.Second):
+		ended = true
+	case <-time.After(killWaits):
+	}
+	again, againErr := h.removeContainers(id)
+	removed += again
+	if removeErr == nil {
+		removeErr = againErr
+	}
+	containers := fmt.Sprintf("%d container(s) it started removed", removed)
+	if removeErr != nil {
+		containers = "its containers could not all be listed or removed: " + removeErr.Error()
+	}
+	if !ended {
 		return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
 			"its outcome is in", id, h.on, containers), nil
 	}
+	h.mu.Lock()
+	outcome, announced := r.outcome, r.announced
+	h.mu.Unlock()
+	if !announced {
+		return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
+			"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
+			containers), nil
+	}
+	return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
+		"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
+}
+
+// removeContainers is one pass of removing what the build left, bounded.
+func (h *holder) removeContainers(id string) (int, error) {
+	cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
+	defer stop()
+	return builder.RemoveContainersOf(cleanup, h.remove, id)
+}
+
+// returned records that the build's work ended, and says whether a kill came first — only then is
+// the build killed; an error it ended with on its own is its own outcome.
+func (h *holder) returned(r *running) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	r.returned = true
+	return r.killed
+}
+
+// said records the outcome announced, and whether it went out, for a kill to answer with.
+func (h *holder) said(r *running, outcome string, announced bool) {
+	h.mu.Lock()
+	r.outcome, r.announced = outcome, announced
+	h.mu.Unlock()
 }
 
 // handlers are the seat's verbs, as this machine answers them.
@@ -313,6 +361,8 @@ func announcement(seat, module, node string) micro.Info {
 		endpoints = append(endpoints, micro.EndpointInfo{
 			Name:    seat + "__" + v.Name,
 			Subject: link.NodeSeatToolSubject(seat, v.Name, node),
+			// The queue group the verbs are served in, as every runtime announces its own.
+			QueueGroup: "seat." + seat,
 			Metadata: map[string]string{
 				"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
 				"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),

cmd/mesh-builder/holder_test.go

@@ -66,9 +66,13 @@ func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
 		!strings.Contains(err.Error(), "build-1") {
 		t.Fatalf("killed another id: %v", err)
 	}
-	// The build's own goroutine: it ends when its context does, as a build's commands do.
+	// The build's own goroutine: it ends when its context does, as a build's commands do, and
+	// announces what came of it.
 	go func() {
 		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, true)
+		}
 		h.end(r)
 	}()
 	answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
@@ -78,14 +82,15 @@ func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
 	if building.Err() == nil {
 		t.Fatal("the build's context was not cancelled")
 	}
-	if !h.wasKilled(r) {
+	if !r.killed {
 		t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
 	}
 	said := answer.(map[string]any)["said"].(string)
-	if !strings.Contains(said, "1 container(s)") || !strings.Contains(said, link.KilledByHand) {
+	if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
 		t.Errorf("kill said %q", said)
 	}
-	if len(removed) != 2 || !strings.Contains(removed[0], "label=mesh.build=build-1") {
+	// Removed at the kill and again once the build had ended: a container made in between is caught.
+	if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
 		t.Errorf("removed %v", removed)
 	}
 	if c := h.current(); c.Running != nil {
@@ -101,7 +106,7 @@ func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
 	}
 	kill := info.Endpoints[1]
 	md := kill.Metadata
-	if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
+	if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
 		md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
 		!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
 		t.Fatalf("kill is announced as %+v", kill)
@@ -111,3 +116,33 @@ func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
 		t.Fatalf("the answer is %d bytes (%v)", len(body), err)
 	}
 }
+
+// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
+// own error is its outcome. One whose outcome could not be announced is not said to be settled.
+func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
+	_, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
+	if killed := h.returned(r); killed {
+		t.Fatal("a build nobody killed reads as killed")
+	}
+	if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
+		t.Fatalf("killed a build that had ended: %v", err)
+	}
+	h.end(r)
+
+	building, cancel := context.WithCancel(context.Background())
+	r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
+	go func() {
+		<-building.Done()
+		if h.returned(r) {
+			h.said(r, link.KilledByHand, false)
+		}
+		h.end(r)
+	}()
+	said, err := h.kill("build-2")
+	if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
+		t.Fatalf("kill said %q (%v)", said, err)
+	}
+}

cmd/mesh-builder/main.go

@@ -247,8 +247,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
 			forgeFrom(), say, request.Seats)
 	}
-	// Only a build the kill ended: one that finished in the moment the kill arrived built, and says so.
-	killed := err != nil && mine != nil && h.wasKilled(mine)
+	// Only a build the kill ended: the kill came before its work did. One that finished — built, or
+	// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
+	killed := false
+	if mine != nil {
+		killed = h.returned(mine) && err != nil
+	}
 	if killed {
 		// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
 		// error it ended with is the kill's consequence, not a fault of the source.
@@ -289,7 +293,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		defer stop()
 		announcing = fresh
 	}
-	if err := work.Announce(announcing, result); err != nil {
+	announceErr := work.Announce(announcing, result)
+	if mine != nil {
+		h.said(mine, result.Failed, announceErr == nil)
+	}
+	if err := announceErr; err != nil {
 		// Said, not fatal: the build happened. A build reported as failed because announcing it
 		// failed is a lie about work that was done — and the request stays unsettled below only if
 		// nothing was said at all, so another machine can try.

cmd/mesh-controller/plan_retry.go

@@ -50,10 +50,20 @@ func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bo
 	if asked == nil {
 		return "", false
 	}
-	return fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
-		strings.Join(pause.Nodes, ", "), now.Sub(*asked).Round(time.Second)), true
+	waited := now.Sub(*asked)
+	said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
+		strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
+	// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
+	// longer than a day is named.
+	if waited > pausedTooLong {
+		said += " — PAUSED OVER A DAY: `resume` takes builds again"
+	}
+	return said, true
 }
 
+// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
+const pausedTooLong = 24 * time.Hour
+
 // awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
 // where the seat being paused changes what a plan says.
 func awaitsABuild(plans []inventory.Plan) bool {

cmd/mesh-controller/queue.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 	"time"
@@ -29,8 +30,12 @@ import (
 // failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
 // rather than waiting for ever on an answer nobody will give.
 
-// holderAsks is how long a holder's verb is waited for.
-const holderAsks = 15 * time.Second
+// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
+// than the holder's worst case (its two passes removing containers and its wait between, 50s).
+const (
+	holderAsks = 15 * time.Second
+	killAnswer = 75 * time.Second
+)
 
 // dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
 func dialTheBus() (*broker.JetStream, error) {
@@ -100,8 +105,11 @@ func queueText(q link.Queue, now time.Time) string {
 		fmt.Fprintln(&b, "\nin flight:")
 		for _, a := range running {
 			where := "taken, not yet said where"
-			if a.On != "" {
+			switch {
+			case a.On != "":
 				where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
+			case a.Was != "":
+				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
 			}
 			fmt.Fprintf(&b, "  %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
 		}
@@ -163,21 +171,19 @@ func cancelCommand(ctx context.Context, args []string) error {
 // cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
 //
 // **In this order, and each step for a reason** (novox/hq ADR 0219):
-//  1. an ask in flight is refused: deleting its message ends nothing a machine is running — that is
-//     `kill`, on the machine running it;
+//  1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
+//     is running — that is `kill`, on the machine running it;
 //  2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
 //  3. for a waiting ask, the worker is read again: one handed out since the queue was read was
-//     taken in the moment of cancelling — the holder that took it either saw the set and ends it
-//     as cancelled, or did not and is building it, and which is for `queue` to say; nothing is
-//     deleted or recorded here, so a build that is running is not recorded as cancelled;
-//  4. the message is deleted by its sequence;
-//  5. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
+//     taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
+//     read the mark first and ends it as cancelled, or is building it;
+//  4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
+//     handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
+//     at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
+//     that took it before the mark is building it, and only `kill` ends that;
+//  5. the message is deleted by its sequence;
+//  6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
 func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
-	// **Refused only where a machine said it is building it.** An ask the worker counts pending that no
-	// machine said it started is either in the moment between a holder's fetch and its start — and
-	// that holder checks the cancelled set first — or one past its deliveries the server has not yet
-	// stopped counting, which happens only when some holder pulls again: with every holder paused it
-	// would stay, cancellable by nothing and killable by nobody.
 	if ask.State == link.AskInFlight && ask.On != "" {
 		return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
 			"ends the build where it runs", ask.ID, ask.On, ask.ID)
@@ -185,14 +191,31 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
 	if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
 		return "", err
 	}
+	withdraw := func() {
+		if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
+			fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
+		}
+	}
 	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
-	if ask.State == link.AskWaiting {
+	switch ask.State {
+	case link.AskWaiting:
 		if taken, err := takenSince(js, worker, ask.Seq); err != nil {
+			withdraw()
 			return "", err
 		} else if taken {
-			return "", fmt.Errorf("%s was taken by a holder as it was cancelled. If the holder saw the cancel "+
-				"it ends it as %s and its outcome says so; otherwise it is building — `queue` says which, "+
-				"and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
+			withdraw()
+			return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
+				"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
+				"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
+		}
+	case link.AskInFlight:
+		if started, err := startedSince(ctx, js, seat, ask); err != nil {
+			withdraw()
+			return "", err
+		} else if started != "" {
+			withdraw()
+			return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
+				"ends it there", ask.ID, started, ask.ID)
 		}
 	}
 	if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
@@ -205,6 +228,38 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
 		"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
 }
 
+// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
+// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
+var holderLooks = 5 * time.Second
+
+// startedSince waits out a holder's look at the cancelled set and says the machine that started the
+// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
+// a start of a build.
+func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
+	select {
+	case <-ctx.Done():
+		return "", ctx.Err()
+	case <-time.After(holderLooks):
+	}
+	since := time.Now().Add(-7 * 24 * time.Hour)
+	if !ask.AskedAt.IsZero() {
+		since = ask.AskedAt.Add(-time.Minute)
+	}
+	heard, err := link.ReadBuildEvents(ctx, js, seat, since)
+	if err != nil {
+		return "", err
+	}
+	s, ok := heard.Started[ask.ID]
+	if !ok || heard.Outcomes[ask.ID] {
+		return "", nil
+	}
+	at, _ := time.Parse(time.RFC3339Nano, s.At)
+	if ask.Started.IsZero() || at.After(ask.Started) {
+		return s.On, nil
+	}
+	return "", nil
+}
+
 // takenSince is whether the worker has handed out the ask at this sequence.
 func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
 	info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
@@ -314,6 +369,19 @@ func rebuildCommand(ctx context.Context, args []string) error {
 	} else if found {
 		source, module = sourceOfBuild(b, entries)
 		path, ref = b.Path, b.Ref
+		// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
+		// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
+		// that commit out over everything since. Building that commit again is `replay`, which says
+		// what it would do and refuses to register it while anything newer is asked or registered.
+		if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
+			ref = followedBranch(e.Source.Ref)
+			follows := ref
+			if follows == "" {
+				follows = "the repository's default branch"
+			}
+			fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
+				"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
+		}
 	} else if e, known := entryNamed(entries, args[0]); known {
 		// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
 		source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
@@ -401,7 +469,25 @@ func replayCommand(ctx context.Context, args []string) error {
 			return err
 		}
 	}
-	if err := replayRefusal(b, module, history, *register, *older); err != nil {
+	// What is asked of the module and not yet answered, when the replay would be registered.
+	var outstanding []string
+	if *register {
+		js, err := dialTheBus()
+		if err != nil {
+			return err
+		}
+		q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
+		js.Close()
+		if err != nil {
+			return err
+		}
+		plans, err := inv.OpenPlans(ctx)
+		if err != nil {
+			return err
+		}
+		outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
+	}
+	if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
 		return err
 	}
 	id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
@@ -420,7 +506,13 @@ func replayCommand(ctx context.Context, args []string) error {
 // current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
 // --register says otherwise, and --register is refused when a newer build of a different commit is
 // registered, unless --older says that is the point.
-func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool) error {
+//
+// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
+// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
+// and their builds — made from newer source — would be recorded and never registered (issue 219
+// orders by ask), the plan waiting on them sending the older commit instead.
+func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
+	outstanding []string) error {
 	if b.Commit == "" {
 		return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
 			"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
@@ -428,6 +520,11 @@ func replayRefusal(b inventory.Build, module string, history []inventory.Build,
 	if older && !register {
 		return errors.New("--older only says what --register may do; a dry run registers nothing")
 	}
+	if register && len(outstanding) > 0 {
+		return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
+			"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
+			orNone(module), strings.Join(outstanding, "; "), b.ID)
+	}
 	if !register || older {
 		return nil
 	}
@@ -487,7 +584,7 @@ func killCommand(ctx context.Context, args []string) error {
 	if heard.Outcomes[id] {
 		return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
 	}
-	answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, 45*time.Second)
+	answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
 	if err != nil {
 		return err
 	}
@@ -581,3 +678,32 @@ func holdersAmong(entries []inventory.Entry, seat string) []string {
 	sort.Strings(out)
 	return out
 }
+
+// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
+// by repository and path, and every open plan holding it not yet built.
+func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
+	var out []string
+	for _, a := range q.Asks {
+		if repositoryMatches(a.Repository, repository) && a.Path == path {
+			out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
+		}
+	}
+	if module == "" {
+		return out
+	}
+	for _, p := range plans {
+		if !p.Open() {
+			continue
+		}
+		for _, tier := range p.Tiers {
+			for _, m := range tier {
+				if m == module {
+					if st := p.Modules[m]; st == nil || st.State != "built" {
+						out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
+					}
+				}
+			}
+		}
+	}
+	return out
+}

cmd/mesh-controller/queue_test.go

@@ -212,31 +212,53 @@ func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
 	newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
 	history := []inventory.Build{newer, old}
 
-	if err := replayRefusal(old, "a", history, false, false); err != nil {
+	if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
 		t.Errorf("a dry run was refused: %v", err)
 	}
-	err := replayRefusal(old, "a", history, true, false)
+	err := replayRefusal(old, "a", history, true, false, nil)
 	if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
 		t.Errorf("registering an older commit than the one registered was not refused: %v", err)
 	}
-	if err := replayRefusal(old, "a", history, true, true); err != nil {
+	if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
 		t.Errorf("--register --older was refused: %v", err)
 	}
-	if err := replayRefusal(newer, "a", history, true, false); err != nil {
+	if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
 		t.Errorf("registering the newest build again was refused: %v", err)
 	}
 	// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
 	same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
 	broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
-	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false); err != nil {
+	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
 		t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
 	}
-	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false); err == nil {
+	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
 		t.Error("a build that recorded no commit was replayed")
 	}
-	if err := replayRefusal(old, "a", history, false, true); err == nil {
+	if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
 		t.Error("--older without --register was taken")
 	}
+	// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
+	// holding it unbuilt, would be replaced by a replay asked now (issue 219).
+	q := link.Queue{Asks: []link.QueuedAsk{
+		{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
+		{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
+		{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
+	}}
+	plans := []inventory.Plan{
+		{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
+		{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
+		{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
+	}
+	outstanding := outstandingFor("a", "novox/a", "", q, plans)
+	if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
+		t.Fatalf("outstanding: %v", outstanding)
+	}
+	if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
+		t.Errorf("registered over an outstanding ask: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
+		t.Errorf("a dry run was refused for what is outstanding: %v", err)
+	}
 	if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
 		t.Errorf("a dry replay does not say what it is: %q", said)
 	}
@@ -266,6 +288,13 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
 		t.Errorf("a plan waiting on a paused seat counted late")
 	}
 
+	longAgo := now.Add(-25 * time.Hour)
+	forgotten := p
+	forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
+	if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan paused over a day reads %q", line)
+	}
+
 	some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
 	if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
 		t.Fatalf("%+v", some)
@@ -644,3 +673,100 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
 		t.Fatalf("asked %v", *asked)
 	}
 }
+
+// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
+// the plan, asked later, never answers it (novox/hq ADR 0219).
+func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := time.Now().UTC().Add(-time.Minute)
+	plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+	planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
+	p, _ := open.inventory.PlanByID(ctx, plan.ID)
+	if p.Modules["a"].State != "asked" {
+		t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
+	}
+	// From the records too: a later build of the module recorded is not the plan's.
+	recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
+	if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
+		t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
+	}
+	planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
+	if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
+		t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
+	}
+}
+
+// rebuild of a build made at a commit asks what the module follows now, never the commit.
+func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var refs []string
+	was := askABuild
+	askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
+		refs = append(refs, ref)
+		return was(c, source, path, ref)
+	}
+	if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
+		Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
+		t.Fatalf("asked %v at %v", *asked, refs)
+	}
+}
+
+// An ask the worker counts out that no machine said it started is cancelled only if no start comes
+// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
+func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	was := holderLooks
+	holderLooks = 300 * time.Millisecond
+	t.Cleanup(func() { holderLooks = was })
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	// The holder that took it says it started, while the cancel waits.
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		_, _ = js.Context().Publish(link.BuildStarted(), started)
+	}()
+	if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
+		t.Fatalf("a build that started was cancelled: %v", err)
+	}
+	if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
+		t.Error("the withdrawn cancel is still marked")
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a withdrawn cancel recorded an outcome")
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -416,7 +416,15 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 		}
 		// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
 		// is, as before, when it was asked at or after the plan's ask (issue 219).
-		if !(id != "" && state.Build == id) && askedBefore(asked, state.AskedAt) {
+		// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
+		// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
+		// build, made from other source, and settling the plan with it would send that. A plan from
+		// before ids were kept is matched as it was: by when the build was asked (issue 219).
+		if state.Build != "" {
+			if state.Build != id {
+				continue
+			}
+		} else if askedBefore(asked, state.AskedAt) {
 			continue
 		}
 		if failed != "" {
@@ -1192,6 +1200,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 		}
 		var outcome *inventory.Build
 		for i := range recorded[m] {
+			// Asked under an id, only that id's record answers (ADR 0219), and it is looked up below.
+			if s.Build != "" {
+				break
+			}
 			b := recorded[m][i]
 			if b.At.Before(*s.AskedAt) {
 				break