mesh-controller/internal/link/checks_nats_test.go

package link

import (
	"context"
	"encoding/json"
	"reflect"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"github.com/novox/mesh-controller/internal/broker"
)

// The checks of pull requests are queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).

func aPullCheck(id string, number int, head, trunk string, ready bool) BuildRequest {
	return BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
		Check: &CheckRequest{Owner: "novox", Repo: "a", Number: number, Trunk: trunk, Ready: ready}}
}

// The subject a check is asked on says its queue and its pull request: ready or other, `<owner>.<repo>.<number>` with
// a dot in a name made an underscore, `group.<name>` for a group's, the id for a try; a build is on the seat's own queue.
func TestACheckIsAskedOnItsQueueBySubject(t *testing.T) {
	for _, c := range []struct {
		r    BuildRequest
		want string
	}{
		{aPullCheck("b1", 7, "h", "t", false), "mesh.seat.node-build-agent.check.other.novox.a.7"},
		{aPullCheck("b2", 7, "h", "t", true), "mesh.seat.node-build-agent.check.ready.novox.a.7"},
		{BuildRequest{ID: "b3", Check: &CheckRequest{Owner: "no.vox", Repo: "a.b", Number: 12}}, "mesh.seat.node-build-agent.check.other.no_vox.a_b.12"},
		{BuildRequest{ID: "b4", Check: &CheckRequest{Owner: "novox", Repo: "a", Group: "g.1", Ready: true}}, "mesh.seat.node-build-agent.check.ready.group.g_1"},
		{BuildRequest{ID: "b5", Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}, "mesh.seat.node-build-agent.check.other.try.b5"},
		{BuildRequest{ID: "b6", Repository: "/r"}, "mesh.seat.node-build-agent.accept.build"},
		{BuildRequest{ID: "b7", Repository: "/r", DryRun: true}, "mesh.seat.node-build-agent.accept.build"},
	} {
		if got := WorkSubjectOf(TheBuildMachine, c.r); got != c.want {
			t.Errorf("%s is asked on %s; want %s", c.r.ID, got, c.want)
		}
	}
	// The retired role has no check queues: its checks wait with its builds.
	if got := WorkSubjectOf(TheBuildMachineBefore, aPullCheck("b8", 7, "h", "t", true)); got != "mesh.seat.mesh-build-machine.accept.build" {
		t.Errorf("a check of the retired role is asked on %s", got)
	}
}

func queueOf(t *testing.T, js *broker.JetStream) Queue {
	t.Helper()
	q, err := ReadQueue(context.Background(), js, TheBuildMachine)
	if err != nil {
		t.Fatal(err)
	}
	return q
}

// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is
// left with one queued check (acceptance 5).
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
		aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
		aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
		aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
		aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	var got []string
	for _, a := range queueOf(t, js).Asks {
		if a.State != AskWaiting {
			t.Errorf("%s is %s", a.ID, a.State)
		}
		got = append(got, a.ID)
	}
	if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
		t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
	}
}

// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the
// order they were asked.
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("other-1", 1, "h1", "t", false),
		aPullCheck("other-2", 2, "h2", "t", false),
		aPullCheck("ready-3", 3, "h3", "t", true),
		{ID: "delivery-4", Repository: "/d"},
		aPullCheck("other-5", 5, "h5", "t", false),
		aPullCheck("ready-6", 6, "h6", "t", true),
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	took := make(chan string, 8)
	machine := MachineOverNATS(js, "anchor")
	defer machine.Close()
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	var order []string
	for len(order) < 6 {
		select {
		case id := <-took:
			order = append(order, id)
		case <-time.After(15 * time.Second):
			t.Fatalf("taken so far: %v", order)
		}
	}
	if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
		t.Fatalf("taken as %v; want %v", order, want)
	}
}

// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module.
func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
	t.Helper()
	api, err := jetstream.New(js.Conn())
	if err != nil {
		t.Fatal(err)
	}
	kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.PullsBucket})
	if err != nil {
		t.Fatal(err)
	}
	return kv
}

func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) {
	t.Helper()
	body, _ := json.Marshal(k)
	if _, err := kv.Put(t.Context(), PullKeyOf("novox", "a", number), body); err != nil {
		t.Fatal(err)
	}
}

// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a
// builder takes it; one for a head the key does not say judged is run.
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
	putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"})                  // asked, not judged
	putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
		aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
		aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	took := make(chan string, 8)
	machine := MachineOverNATS(js, "anchor")
	defer machine.Close()
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	var ran []string
	for len(ran) < 2 {
		select {
		case id := <-took:
			ran = append(ran, id)
		case <-time.After(15 * time.Second):
			t.Fatalf("run so far: %v", ran)
		}
	}
	if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
		t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
	}
	time.Sleep(300 * time.Millisecond)
	if n := len(queueOf(t, js).Asks); n != 0 {
		t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
	}
}

// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a
// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all.
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	moved := make(chan PullKey, 4)
	req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
	end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end()
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
	select {
	case k := <-moved:
		t.Fatalf("stopped by a key that moved nothing: %+v", k)
	case <-time.After(500 * time.Millisecond):
	}
	putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
	select {
	case k := <-moved:
		if k.Head != "head-7-111111111" {
			t.Errorf("stopped by %+v", k)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("a key whose head moved did not stop the check")
	}

	// And the trunk commit.
	req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
	end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end2()
	putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
	select {
	case <-moved:
	case <-time.After(5 * time.Second):
		t.Fatal("a key whose trunk commit moved did not stop the check")
	}

	// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
	// have is the pull request's own.
	group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
		Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
	end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end3()
	putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
	select {
	case k := <-moved:
		t.Fatalf("a group's check was stopped by %+v", k)
	case <-time.After(500 * time.Millisecond):
	}
}

func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
	k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"}
	if k.Moved("aaaaaaaaaaaa", "tttttttttttt") || !k.Moved("bbbbbbbbbbbbbbbb", "tttttttttttt") || !k.Moved("aaaaaaaaaaaa", "uuuuuuuuuuuuuu") {
		t.Error("moved is wrong")
	}
	if !k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111") || k.Judged("aaaaaaaaaaaa1111", "") || (PullKey{Head: "a", Trunk: "t"}).Judged("a", "t") {
		t.Error("judged is wrong")
	}
}

// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an
// ask is removed from that stream.
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("other-1", 1, "h1", "t", false),
		aPullCheck("ready-2", 2, "h2", "t", true),
		{ID: "delivery-3", Repository: "/d"},
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	q := queueOf(t, js)
	var ids, streams []string
	for _, a := range q.Asks {
		ids, streams = append(ids, a.ID), append(streams, a.Stream)
	}
	if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
		t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
	}
	if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
		t.Fatalf("streams %v; want %v", streams, want)
	}
	ready, _ := q.Find("ready-2")
	if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
		t.Errorf("the ready check is handed out by %s", w.Name)
	}
	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
		t.Fatalf("deleted %d, %v", removed, err)
	}
	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
		t.Fatalf("deleted again: %d, %v", removed, err)
	}
	if n := len(queueOf(t, js).Asks); n != 2 {
		t.Errorf("%d asks left; want 2", n)
	}
}

// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
// (the mesh's is two minutes): a key not renewed within it is gone.
func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue {
	t.Helper()
	api, err := jetstream.New(js.Conn())
	if err != nil {
		t.Fatal(err)
	}
	kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age})
	if err != nil {
		t.Fatal(err)
	}
	return kv
}

// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	aKeyBucket(t, js)
	aRunningBucket(t, js, 2*time.Minute)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	running := make(chan string, 4)
	release := make(chan struct{})
	a := MachineOverNATS(js, "builder-a")
	defer a.Close()
	go func() {
		_ = a.Take(ctx, func(ctx context.Context, work Build) {
			running <- work.Request().ID
			<-release
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case <-running:
	case <-time.After(10 * time.Second):
		t.Fatal("builder a took nothing")
	}
	// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
	if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	b := MachineOverNATS(js, "builder-b")
	defer b.Close()
	took := make(chan string, 4)
	go func() {
		_ = b.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
			_ = work.Done()
		})
	}()
	select {
	case id := <-took:
		t.Fatalf("builder b took %s, which builder a is running a check of", id)
	case <-time.After(3 * time.Second):
	}
	if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
		t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
	}
	// Another trunk commit is another check.
	if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		if id != "moved" {
			t.Fatalf("builder b took %s", id)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("a check against another trunk commit was dropped too")
	}
	close(release)
}

// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
// running a check leaves nothing that holds the next one back for longer than the bucket's age.
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	aKeyBucket(t, js)
	kv := aRunningBucket(t, js, 2*time.Second)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
	if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
		t.Fatal(err)
	}
	asks := BuildsOn(js, TheBuildMachine)
	took := make(chan string, 4)
	b := MachineOverNATS(js, "builder-b")
	defer b.Close()
	go func() {
		_ = b.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		t.Fatalf("%s was taken while the key was fresh", id)
	case <-time.After(1200 * time.Millisecond):
	}
	time.Sleep(2500 * time.Millisecond) // the key was not renewed
	if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		if id != "after-it-aged" {
			t.Fatalf("took %s", id)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("the request was not taken after the key aged out")
	}
}

// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
// ends, whatever the outcome.
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aRunningBucket(t, js, 2*time.Second)
	was := heartbeatWhileWorking
	heartbeatWhileWorking = 400 * time.Millisecond
	t.Cleanup(func() { heartbeatWhileWorking = was })
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	running := make(chan struct{}, 1)
	release := make(chan struct{})
	m := MachineOverNATS(js, "builder-a")
	defer m.Close()
	go func() {
		_ = m.Take(ctx, func(ctx context.Context, work Build) {
			running <- struct{}{}
			<-release
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	<-running
	key := PullKeyOf("novox", "a", 7)
	e, err := kv.Get(ctx, key)
	if err != nil {
		t.Fatalf("no key while the check runs: %v", err)
	}
	var got CheckRunning
	_ = json.Unmarshal(e.Value(), &got)
	if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
		t.Errorf("the key says %+v", got)
	}
	time.Sleep(3 * time.Second) // longer than the bucket's age
	if _, err := kv.Get(ctx, key); err != nil {
		t.Fatalf("the key aged out under a check still running: %v", err)
	}
	close(release)
	for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
		if _, err := kv.Get(ctx, key); err != nil {
			break
		}
		if time.Now().After(end) {
			t.Fatal("the key stays after the check ended")
		}
	}
}

// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and
// trunk commit must run (issue 974, review 2).
func TestAnErrorVerdictJudgesNothing(t *testing.T) {
	for verdict, judged := range map[string]bool{"pass": true, "warning": true, "fail": true, "error": false, "": false, "pending": false} {
		k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: verdict}
		if got := k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111"); got != judged {
			t.Errorf("a key with verdict %q judged = %v; want %v", verdict, got, judged)
		}
	}
}

// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and
// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it.
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	aRunningBucket(t, js, 2*time.Minute)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	outcomes := make(chan BuildResult, 8)
	watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
		var r BuildResult
		if json.Unmarshal(m.Data, &r) == nil {
			outcomes <- r
		}
	})
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = watching.Unsubscribe() }()

	ran := make(chan string, 4)
	m := MachineOverNATS(js, "builder-a")
	defer m.Close()
	go func() {
		_ = m.Take(ctx, func(ctx context.Context, work Build) {
			building, cancel := context.WithCancel(ctx)
			defer cancel()
			ran <- work.Request().ID
			stopped := make(chan struct{}, 1)
			end, err := work.(interface {
				WatchPull(context.Context, func(PullKey)) (func(), error)
			}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
			if err != nil {
				t.Error(err)
				return
			}
			defer end()
			if work.Request().ID == "older" {
				select {
				case <-stopped:
					_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
				case <-time.After(15 * time.Second):
					_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
				}
			} else {
				_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
			}
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	if id := <-ran; id != "older" {
		t.Fatalf("took %s", id)
	}
	time.Sleep(300 * time.Millisecond) // the watch is made
	// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
	if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-ran:
		if id != "newer" {
			t.Fatalf("took %s", id)
		}
	case <-time.After(15 * time.Second):
		t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
	}
	got := map[string]string{}
	for len(got) < 2 {
		select {
		case r := <-outcomes:
			got[r.ID] = r.Failed
		case <-time.After(10 * time.Second):
			t.Fatalf("outcomes so far: %v", got)
		}
	}
	if got["older"] != SupersededByKey || got["newer"] != "" {
		t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
	}
}

// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again
// when the worker's wait is up.
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
	for _, ready := range []bool{false, true} {
		js := aBusWithTheBuildRole(t)
		name, worker := "other", broker.ChecksOtherWorker
		if ready {
			name, worker = "ready", broker.ChecksReadyWorker
		}
		// A worker whose wait is a second, as the mesh's is a minute.
		for _, w := range broker.CheckWorkers() {
			if w.Name == worker {
				w.AckWaitSeconds = 1
				if err := js.EnsureConsumer(w); err != nil {
					t.Fatal(err)
				}
			}
		}
		ctx, stop := context.WithCancel(t.Context())
		if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
			t.Fatal(err)
		}
		// The first builder takes it and dies: its pull is fetched, never acknowledged.
		if _, err := fetchOne(js, worker); err != nil {
			t.Fatal(err)
		}
		took := make(chan BuildRequest, 2)
		second := MachineOverNATS(js, "builder-b")
		go func() {
			_ = second.Take(ctx, func(ctx context.Context, work Build) {
				took <- work.Request()
				_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
				_ = work.Done()
			})
		}()
		select {
		case r := <-took:
			if r.ID != "dies-"+name {
				t.Errorf("%s: took %s", name, r.ID)
			}
		case <-time.After(15 * time.Second):
			t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
		}
		second.Close()
		stop()
	}
}

// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would.
func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) {
	for _, w := range broker.CheckWorkers() {
		if w.Name != worker {
			continue
		}
		sub, err := js.Context().PullSubscribe(w.Filters[0], w.Name, nats.Bind(w.Stream, w.Name), nats.ManualAck())
		if err != nil {
			return nil, err
		}
		got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
		if err != nil {
			return nil, err
		}
		return got[0], nil
	}
	return nil, nil
}