mesh-controller/internal/link/checks_nats_integration_test.go

//go:build integration

package link

import (
	"context"
	"encoding/json"
	"reflect"
	"testing"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/novox/mesh-controller/internal/broker"
)

func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
		aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
		aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
		aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
		aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	var got []string
	for _, a := range queueOf(t, js).Asks {
		if a.State != AskWaiting {
			t.Errorf("%s is %s", a.ID, a.State)
		}
		got = append(got, a.ID)
	}
	if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
		t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
	}
}

func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("other-1", 1, "h1", "t", false),
		aPullCheck("other-2", 2, "h2", "t", false),
		aPullCheck("ready-3", 3, "h3", "t", true),
		{ID: "delivery-4", Repository: "/d"},
		aPullCheck("other-5", 5, "h5", "t", false),
		aPullCheck("ready-6", 6, "h6", "t", true),
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	took := make(chan string, 8)
	machine := MachineOverNATS(js, "anchor")
	defer machine.Close()
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	var order []string
	for len(order) < 6 {
		select {
		case id := <-took:
			order = append(order, id)
		case <-time.After(15 * time.Second):
			t.Fatalf("taken so far: %v", order)
		}
	}
	if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
		t.Fatalf("taken as %v; want %v", order, want)
	}
}

func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
	putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"})                  // asked, not judged
	putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
		aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
		aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	took := make(chan string, 8)
	machine := MachineOverNATS(js, "anchor")
	defer machine.Close()
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	var ran []string
	for len(ran) < 2 {
		select {
		case id := <-took:
			ran = append(ran, id)
		case <-time.After(15 * time.Second):
			t.Fatalf("run so far: %v", ran)
		}
	}
	if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
		t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
	}
	time.Sleep(300 * time.Millisecond)
	if n := len(queueOf(t, js).Asks); n != 0 {
		t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
	}
}

func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	moved := make(chan PullKey, 4)
	req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
	end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end()
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
	select {
	case k := <-moved:
		t.Fatalf("stopped by a key that moved nothing: %+v", k)
	case <-time.After(500 * time.Millisecond):
	}
	putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
	select {
	case k := <-moved:
		if k.Head != "head-7-111111111" {
			t.Errorf("stopped by %+v", k)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("a key whose head moved did not stop the check")
	}

	// And the trunk commit.
	req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
	end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end2()
	putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
	select {
	case <-moved:
	case <-time.After(5 * time.Second):
		t.Fatal("a key whose trunk commit moved did not stop the check")
	}

	// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
	// have is the pull request's own.
	group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
		Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
	end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
	if err != nil {
		t.Fatal(err)
	}
	defer end3()
	putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
	select {
	case k := <-moved:
		t.Fatalf("a group's check was stopped by %+v", k)
	case <-time.After(500 * time.Millisecond):
	}
}

func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	asks := BuildsOn(js, TheBuildMachine)
	for _, r := range []BuildRequest{
		aPullCheck("other-1", 1, "h1", "t", false),
		aPullCheck("ready-2", 2, "h2", "t", true),
		{ID: "delivery-3", Repository: "/d"},
	} {
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	q := queueOf(t, js)
	var ids, streams []string
	for _, a := range q.Asks {
		ids, streams = append(ids, a.ID), append(streams, a.Stream)
	}
	if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
		t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
	}
	if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
		t.Fatalf("streams %v; want %v", streams, want)
	}
	ready, _ := q.Find("ready-2")
	if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
		t.Errorf("the ready check is handed out by %s", w.Name)
	}
	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
		t.Fatalf("deleted %d, %v", removed, err)
	}
	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
		t.Fatalf("deleted again: %d, %v", removed, err)
	}
	if n := len(queueOf(t, js).Asks); n != 2 {
		t.Errorf("%d asks left; want 2", n)
	}
}

func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	aKeyBucket(t, js)
	aRunningBucket(t, js, 2*time.Minute)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	running := make(chan string, 4)
	release := make(chan struct{})
	a := MachineOverNATS(js, "builder-a")
	defer a.Close()
	go func() {
		_ = a.Take(ctx, func(ctx context.Context, work Build) {
			running <- work.Request().ID
			<-release
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case <-running:
	case <-time.After(10 * time.Second):
		t.Fatal("builder a took nothing")
	}
	// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
	if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	b := MachineOverNATS(js, "builder-b")
	defer b.Close()
	took := make(chan string, 4)
	go func() {
		_ = b.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
			_ = work.Done()
		})
	}()
	select {
	case id := <-took:
		t.Fatalf("builder b took %s, which builder a is running a check of", id)
	case <-time.After(3 * time.Second):
	}
	if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
		t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
	}
	// Another trunk commit is another check.
	if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		if id != "moved" {
			t.Fatalf("builder b took %s", id)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("a check against another trunk commit was dropped too")
	}
	close(release)
}

func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	aKeyBucket(t, js)
	kv := aRunningBucket(t, js, 2*time.Second)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
	if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
		t.Fatal(err)
	}
	asks := BuildsOn(js, TheBuildMachine)
	took := make(chan string, 4)
	b := MachineOverNATS(js, "builder-b")
	defer b.Close()
	go func() {
		_ = b.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().ID
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		t.Fatalf("%s was taken while the key was fresh", id)
	case <-time.After(1200 * time.Millisecond):
	}
	time.Sleep(2500 * time.Millisecond) // the key was not renewed
	if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-took:
		if id != "after-it-aged" {
			t.Fatalf("took %s", id)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("the request was not taken after the key aged out")
	}
}

func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aRunningBucket(t, js, 2*time.Second)
	was := heartbeatWhileWorking
	heartbeatWhileWorking = 400 * time.Millisecond
	t.Cleanup(func() { heartbeatWhileWorking = was })
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	running := make(chan struct{}, 1)
	release := make(chan struct{})
	m := MachineOverNATS(js, "builder-a")
	defer m.Close()
	go func() {
		_ = m.Take(ctx, func(ctx context.Context, work Build) {
			running <- struct{}{}
			<-release
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
			_ = work.Done()
		})
	}()
	if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	<-running
	key := PullKeyOf("novox", "a", 7)
	e, err := kv.Get(ctx, key)
	if err != nil {
		t.Fatalf("no key while the check runs: %v", err)
	}
	var got CheckRunning
	_ = json.Unmarshal(e.Value(), &got)
	if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
		t.Errorf("the key says %+v", got)
	}
	time.Sleep(3 * time.Second) // longer than the bucket's age
	if _, err := kv.Get(ctx, key); err != nil {
		t.Fatalf("the key aged out under a check still running: %v", err)
	}
	close(release)
	for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
		if _, err := kv.Get(ctx, key); err != nil {
			break
		}
		if time.Now().After(end) {
			t.Fatal("the key stays after the check ended")
		}
	}
}

func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	kv := aKeyBucket(t, js)
	aRunningBucket(t, js, 2*time.Minute)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	outcomes := make(chan BuildResult, 8)
	watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
		var r BuildResult
		if json.Unmarshal(m.Data, &r) == nil {
			outcomes <- r
		}
	})
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = watching.Unsubscribe() }()

	ran := make(chan string, 4)
	m := MachineOverNATS(js, "builder-a")
	defer m.Close()
	go func() {
		_ = m.Take(ctx, func(ctx context.Context, work Build) {
			building, cancel := context.WithCancel(ctx)
			defer cancel()
			ran <- work.Request().ID
			stopped := make(chan struct{}, 1)
			end, err := work.(interface {
				WatchPull(context.Context, func(PullKey)) (func(), error)
			}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
			if err != nil {
				t.Error(err)
				return
			}
			defer end()
			if work.Request().ID == "older" {
				select {
				case <-stopped:
					_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
				case <-time.After(15 * time.Second):
					_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
				}
			} else {
				_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
			}
			_ = work.Done()
		})
	}()
	if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
		t.Fatal(err)
	}
	if id := <-ran; id != "older" {
		t.Fatalf("took %s", id)
	}
	time.Sleep(300 * time.Millisecond) // the watch is made
	// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
	putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
	if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
		t.Fatal(err)
	}
	select {
	case id := <-ran:
		if id != "newer" {
			t.Fatalf("took %s", id)
		}
	case <-time.After(15 * time.Second):
		t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
	}
	got := map[string]string{}
	for len(got) < 2 {
		select {
		case r := <-outcomes:
			got[r.ID] = r.Failed
		case <-time.After(10 * time.Second):
			t.Fatalf("outcomes so far: %v", got)
		}
	}
	if got["older"] != SupersededByKey || got["newer"] != "" {
		t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
	}
}

func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
	for _, ready := range []bool{false, true} {
		js := aBusWithTheBuildRole(t)
		name, worker := "other", broker.ChecksOtherWorker
		if ready {
			name, worker = "ready", broker.ChecksReadyWorker
		}
		// A worker whose wait is a second, as the mesh's is a minute.
		for _, w := range broker.CheckWorkers() {
			if w.Name == worker {
				w.AckWaitSeconds = 1
				if err := js.EnsureConsumer(w); err != nil {
					t.Fatal(err)
				}
			}
		}
		ctx, stop := context.WithCancel(t.Context())
		if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
			t.Fatal(err)
		}
		// The first builder takes it and dies: its pull is fetched, never acknowledged.
		if _, err := fetchOne(js, worker); err != nil {
			t.Fatal(err)
		}
		took := make(chan BuildRequest, 2)
		second := MachineOverNATS(js, "builder-b")
		go func() {
			_ = second.Take(ctx, func(ctx context.Context, work Build) {
				took <- work.Request()
				_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
				_ = work.Done()
			})
		}()
		select {
		case r := <-took:
			if r.ID != "dies-"+name {
				t.Errorf("%s: took %s", name, r.ID)
			}
		case <-time.After(15 * time.Second):
			t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
		}
		second.Close()
		stop()
	}
}