Keep a replay from moving the mesh backwards, and tighten the queue's edges (review of hq ADR 0219)

A registered replay asked now outranked newer asks of its module, and a plan
took any later outcome as its answer. A replay is now refused while the module
is asked anywhere, a plan module asked under an id is answered by that id
alone, and a rebuild of a commit asks what the module follows. An ask handed
back after a restart no longer reads as dead; a cancel that meets a start is
withdrawn; pause holds for an ask fetched as it lands; kill removes containers
before and after the build ends and says whether its outcome went out; a
holder may say only its own machine is paused.

internal/link/queue_test.go

@@ -3,6 +3,7 @@ package link
 import (
 	"context"
 	"encoding/json"
+	"sync"
 	"testing"
 	"time"
 
@@ -35,9 +36,9 @@ func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) {
 		},
 		Outcomes: map[string]bool{"done": true},
 	}
-	got := ClassifyQueue(asks, 4, 3, heard)
+	got := ClassifyQueue(asks, 4, 2, heard)
 	want := map[string]string{"dead-1": AskDead, "running-g": AskInFlight, "running-a": AskInFlight,
-		"unsaid": AskInFlight, "waiting-1": AskWaiting, "waiting-2": AskWaiting}
+		"unsaid": AskDead, "waiting-1": AskWaiting, "waiting-2": AskWaiting}
 	for _, a := range got {
 		if a.State != want[a.ID] {
 			t.Errorf("%s is %s, want %s", a.ID, a.State, want[a.ID])
@@ -48,16 +49,45 @@ func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) {
 		t.Errorf("an ask in flight does not say where: %+v", a)
 	}
 
-	// **The worker's count bounds it**: with two pending, the machine whose start is oldest died with
-	// its ask, and the ask nobody said is not in flight either.
-	got = ClassifyQueue(asks, 4, 2, heard)
+	// **The worker's count bounds it**: with one pending, the machine whose start is oldest died
+	// with its ask.
+	got = ClassifyQueue(asks, 4, 1, heard)
 	q = Queue{Asks: got}
-	if a, _ := q.Find("running-g"); a.State != AskInFlight {
-		t.Errorf("running-g is %s", a.State)
+	if a, _ := q.Find("running-a"); a.State != AskInFlight {
+		t.Errorf("running-a is %s", a.State)
+	}
+	if a, _ := q.Find("running-g"); a.State != AskDead || a.On != "" {
+		t.Errorf("past the worker's count running-g is %s on %q", a.State, a.On)
+	}
+	// **Handed back after a holder restarted mid-build**: started on g14, g14 then started something
+	// else, and the worker still counts it — it waits to be handed out again, and is not dead. With
+	// no more asks behind the worker than it counts pending, none is dead.
+	restarted := []QueuedAsk{{Seq: 1, ID: "dead-1"}, {Seq: 2, ID: "running-g"}}
+	for _, a := range ClassifyQueue(restarted, 2, 2, heard) {
+		if a.State != AskInFlight {
+			t.Errorf("%s is %s with two behind the worker and two pending", a.ID, a.State)
+		}
+		if a.ID == "dead-1" && (a.On != "" || a.Was != "g14") {
+			t.Errorf("an ask handed back reads on %q, was on %q", a.On, a.Was)
+		}
+	}
+	// The handed-back one takes a leftover slot before an ask nobody said started.
+	got = ClassifyQueue(asks, 4, 4, heard)
+	q = Queue{Asks: got}
+	for _, id := range []string{"dead-1", "running-g", "running-a", "unsaid"} {
+		if a, _ := q.Find(id); a.State != AskInFlight {
+			t.Errorf("with four pending %s is %s", id, a.State)
+		}
+	}
+	got = ClassifyQueue(asks, 4, 3, heard)
+	q = Queue{Asks: got}
+	if a, _ := q.Find("dead-1"); a.State != AskInFlight {
+		t.Errorf("the handed-back ask lost its slot to one nobody said: %s", a.State)
 	}
 	if a, _ := q.Find("unsaid"); a.State != AskDead {
-		t.Errorf("an ask no start explains, past the worker's count, is %s", a.State)
+		t.Errorf("unsaid is %s", a.State)
 	}
+
 	// None pending: everything behind the worker is dead, and names no machine.
 	for _, a := range ClassifyQueue(asks, 4, 0, heard) {
 		if a.Seq <= 4 && (a.State != AskDead || a.On != "") {
@@ -265,3 +295,42 @@ func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) {
 		t.Fatalf("the waiter heard %+v (%v)", result, err)
 	}
 }
+
+// Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219).
+func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) {
+	js := aBusWithACancelledSet(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	// Not paused when it asks; paused by the time the answer is read.
+	var looks int
+	var mu sync.Mutex
+	paused := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		looks++
+		return looks > 1
+	}
+	took := make(chan string, 1)
+	machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused})
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID })
+	}()
+	time.Sleep(200 * time.Millisecond)
+	body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"})
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		t.Fatalf("a holder paused as its pull was answered built %s", id)
+	case <-time.After(2 * time.Second):
+	}
+	q, err := ReadQueue(ctx, js, TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" {
+		t.Fatalf("the ask is not back in the queue: %+v", q.Asks)
+	}
+}