issue 974: the builders say the checks they run in a bucket that ages, and a builder drops a waiting request for a head and trunk commit another is running

2026-10-17 14:10:26 +02:00
parent 357e08a4f6
commit bd257afbd5
10 changed files with 380 additions and 9 deletions
@@ -307,3 +307,180 @@ func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testi
 		t.Errorf("%d asks left; want 2", n)
 	}
 }
+
+// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
+// (the mesh's is two minutes): a key not renewed within it is gone.
+func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue {
+	t.Helper()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return kv
+}
+
+// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
+// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
+func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	aKeyBucket(t, js)
+	aRunningBucket(t, js, 2*time.Minute)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+	running := make(chan string, 4)
+	release := make(chan struct{})
+	a := MachineOverNATS(js, "builder-a")
+	defer a.Close()
+	go func() {
+		_ = a.Take(ctx, func(ctx context.Context, work Build) {
+			running <- work.Request().ID
+			<-release
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-running:
+	case <-time.After(10 * time.Second):
+		t.Fatal("builder a took nothing")
+	}
+	// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
+	if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	b := MachineOverNATS(js, "builder-b")
+	defer b.Close()
+	took := make(chan string, 4)
+	go func() {
+		_ = b.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	select {
+	case id := <-took:
+		t.Fatalf("builder b took %s, which builder a is running a check of", id)
+	case <-time.After(3 * time.Second):
+	}
+	if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
+		t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
+	}
+	// Another trunk commit is another check.
+	if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		if id != "moved" {
+			t.Fatalf("builder b took %s", id)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("a check against another trunk commit was dropped too")
+	}
+	close(release)
+}
+
+// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
+// running a check leaves nothing that holds the next one back for longer than the bucket's age.
+func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	aKeyBucket(t, js)
+	kv := aRunningBucket(t, js, 2*time.Second)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
+	if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
+		t.Fatal(err)
+	}
+	asks := BuildsOn(js, TheBuildMachine)
+	took := make(chan string, 4)
+	b := MachineOverNATS(js, "builder-b")
+	defer b.Close()
+	go func() {
+		_ = b.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		t.Fatalf("%s was taken while the key was fresh", id)
+	case <-time.After(1200 * time.Millisecond):
+	}
+	time.Sleep(2500 * time.Millisecond) // the key was not renewed
+	if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case id := <-took:
+		if id != "after-it-aged" {
+			t.Fatalf("took %s", id)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("the request was not taken after the key aged out")
+	}
+}
+
+// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
+// ends, whatever the outcome.
+func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	kv := aRunningBucket(t, js, 2*time.Second)
+	was := heartbeatWhileWorking
+	heartbeatWhileWorking = 400 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = was })
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	running := make(chan struct{}, 1)
+	release := make(chan struct{})
+	m := MachineOverNATS(js, "builder-a")
+	defer m.Close()
+	go func() {
+		_ = m.Take(ctx, func(ctx context.Context, work Build) {
+			running <- struct{}{}
+			<-release
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
+		t.Fatal(err)
+	}
+	<-running
+	key := PullKeyOf("novox", "a", 7)
+	e, err := kv.Get(ctx, key)
+	if err != nil {
+		t.Fatalf("no key while the check runs: %v", err)
+	}
+	var got CheckRunning
+	_ = json.Unmarshal(e.Value(), &got)
+	if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
+		t.Errorf("the key says %+v", got)
+	}
+	time.Sleep(3 * time.Second) // longer than the bucket's age
+	if _, err := kv.Get(ctx, key); err != nil {
+		t.Fatalf("the key aged out under a check still running: %v", err)
+	}
+	close(release)
+	for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
+		if _, err := kv.Get(ctx, key); err != nil {
+			break
+		}
+		if time.Now().After(end) {
+			t.Fatal("the key stays after the check ended")
+		}
+	}
+}