Keep a replay from moving the mesh backwards, and tighten the queue's edges (review of hq ADR 0219)

A registered replay asked now outranked newer asks of its module, and a plan
took any later outcome as its answer. A replay is now refused while the module
is asked anywhere, a plan module asked under an id is answered by that id
alone, and a rebuild of a commit asks what the module follows. An ask handed
back after a restart no longer reads as dead; a cancel that meets a start is
withdrawn; pause holds for an ask fetched as it lands; kill removes containers
before and after the build ends and says whether its outcome went out; a
holder may say only its own machine is paused.

internal/link/queue.go

@@ -62,8 +62,11 @@ type QueuedAsk struct {
 	Ref        string    `json:"ref,omitempty"`
 	AskedAt    time.Time `json:"asked-at,omitempty"`
 	State      string    `json:"state"`
-	// On and Started are the holder that took an in-flight ask and when, from its started event.
+	// On and Started are the holder building an in-flight ask and when it started, from its started
+	// event. Was is the holder that started an in-flight ask and is no longer building it — handed
+	// back, to be handed out again — with Started its start there.
 	On      string    `json:"on,omitempty"`
+	Was     string    `json:"was-on,omitempty"`
 	Started time.Time `json:"started,omitempty"`
 }
 
@@ -109,19 +112,24 @@ type BuildHeard struct {
 // ClassifyQueue says which state each ask is in. Pure: the stream's asks in sequence order, what the
 // worker says, and what the seat's events said.
 //
-// **In flight is what a machine is building now.** A holder builds one ask at a time (ADR 0190), so
-// what a machine is building is its latest start, if no outcome has been heard for it. An ask
-// behind the worker that is some machine's current build is in flight; at most AckPending of them
-// are, newest start first — a machine that died mid-build has a latest start forever, and the
-// worker's own count is what bounds that. An ask behind the worker that no machine has said it
-// started is in flight only while the worker counts more pending than starts explain (taken, and
-// not yet said); otherwise it is dead.
+// **In flight is what the worker counts handed out and unsettled**, at most AckPending of the asks
+// behind it, given out in this order:
+//
+//  1. what a machine is building now — its latest start, no outcome heard (a holder builds one ask
+//     at a time, ADR 0190), newest start first;
+//  2. what a machine started and is no longer building — a holder restarted mid-build, the ask
+//     handed back and waiting to be handed out again while it has deliveries left — newest start
+//     first, naming the machine it was last on;
+//  3. what was taken and not yet said started.
+//
+// Only what is left after that is dead: so nothing behind the worker is dead while there are no more
+// of them than the worker counts pending. A machine that died mid-build keeps a latest start for
+// ever; the worker's count is what says the ask was handed out as often as it may be.
 //
 // **The worker's count is the server's, and it lags in one case**: an ask handed out its last time
 // and handed back is still counted pending until some holder pulls again, when the server finds it
 // past its deliveries and lets it go. Holders pull whenever they are idle, so this is a moment —
 // except while every holder is paused, when such an ask reads as in flight with no machine named.
-// Cancel takes an ask in flight that no machine said it started, for exactly that reason.
 func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard BuildHeard) []QueuedAsk {
 	// Each machine's latest start.
 	latest := map[string]BuildStart{}
@@ -140,7 +148,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
 
 	out := make([]QueuedAsk, len(asks))
 	copy(out, asks)
-	var running, unsaid []int
+	var running, handedBack, unsaid []int
 	for i := range out {
 		a := &out[i]
 		if a.Seq > delivered {
@@ -153,28 +161,29 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
 			running = append(running, i)
 			continue
 		}
-		if _, ever := heard.Started[a.ID]; !ever {
-			unsaid = append(unsaid, i)
-		}
-	}
-	sort.SliceStable(running, func(x, y int) bool { return out[running[x]].Started.After(out[running[y]].Started) })
-	slots := ackPending
-	for _, i := range running {
-		if slots == 0 {
-			// A machine's latest start the worker no longer counts: it died with the ask, and the
-			// ask was handed out as often as it may be.
-			out[i].On, out[i].Started = "", time.Time{}
+		if s, ever := heard.Started[a.ID]; ever {
+			a.Was, a.Started = s.On, startedAt(s)
+			handedBack = append(handedBack, i)
 			continue
 		}
-		out[i].State = AskInFlight
-		slots--
+		unsaid = append(unsaid, i)
 	}
-	for _, i := range unsaid {
-		if slots == 0 {
-			break
+	newestFirst := func(ix []int) {
+		sort.SliceStable(ix, func(x, y int) bool { return out[ix[x]].Started.After(out[ix[y]].Started) })
+	}
+	newestFirst(running)
+	newestFirst(handedBack)
+	slots := ackPending
+	for _, group := range [][]int{running, handedBack, unsaid} {
+		for _, i := range group {
+			if slots == 0 {
+				// Past what the worker counts: handed out as often as it may be.
+				out[i].On, out[i].Started = "", time.Time{}
+				continue
+			}
+			out[i].State = AskInFlight
+			slots--
 		}
-		out[i].State = AskInFlight
-		slots--
 	}
 	return out
 }
@@ -336,6 +345,23 @@ func MarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) e
 	return err
 }
 
+// UnmarkCancelled takes an ask's id out of the cancelled set: a cancel withdrawn, because the ask
+// was taken as it was being cancelled.
+func UnmarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error {
+	if !safeKey.MatchString(id) {
+		return nil
+	}
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat))
+	if err != nil {
+		return err
+	}
+	return kv.Delete(ctx, id)
+}
+
 // IsCancelled asks the seat's cancelled set whether an ask was cancelled: one direct read of one key,
 // which is all a holder is granted of it.
 func IsCancelled(conn *nats.Conn, seat, id string) (bool, error) {