mesh-controller/internal/link/queue_order_test.go

package link

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
)

// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
// first come, first served within a kind.

func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
	r := BuildRequest{ID: id, Repository: "/r"}
	if check {
		r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
	}
	return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
}

func idsOf(asks []QueuedAsk) []string {
	out := []string{}
	for _, a := range asks {
		out = append(out, a.ID)
	}
	return out
}

func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
	now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
	waiting := []QueuedAsk{
		anAsk(1, "check-a", now.Add(-3*time.Minute), true),
		anAsk(2, "check-b", now.Add(-2*time.Minute), true),
		anAsk(3, "delivery-d", now.Add(-time.Minute), false),
	}
	want := []string{"delivery-d", "check-a", "check-b"}
	if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
		t.Fatalf("handed out as %v, want %v", got, want)
	}
	// Only the checks ahead of it move behind it; the delivery stays where it is.
	if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
		t.Fatalf("moved %v", got)
	}
}

func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
	now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
	// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
	waiting := []QueuedAsk{
		anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
		anAsk(6, "delivery-later", now.Add(-time.Minute), false),
		anAsk(7, "check-old", now.Add(-11*time.Minute), true),
		anAsk(8, "check-young", now.Add(-9*time.Minute), true),
	}
	want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
	if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
		t.Fatalf("handed out as %v, want %v", got, want)
	}
	if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
		t.Fatalf("moved %v", got)
	}
}

func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
	for _, r := range []BuildRequest{
		{},                                      // unreadable: nothing was read from it
		{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
		{ID: "build-2", DryRun: true},           // looked at, never delivered
	} {
		if got := KindOf(r); got != KindCheck {
			t.Errorf("%+v is a %s", r, got)
		}
	}
	if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
		t.Errorf("a build is a %s", got)
	}
}

// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx, stop := context.WithCancel(context.Background())
	defer stop()
	asks := BuildsOn(js, TheBuildMachine)
	now := time.Now()
	for i, r := range []BuildRequest{
		{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
		{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
		{ID: NewBuildID(now), Repository: "/d"},
	} {
		if i == 2 {
			// And one nobody can read, whose kind cannot be told: it waits with the checks.
			if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
				t.Fatal(err)
			}
		}
		if err := asks.Ask(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	// The acting controller's loop puts it in order, told by the asks.
	go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
		func(string, ...any) {})
	time.Sleep(500 * time.Millisecond)
	took := make(chan string, 4)
	machine := MachineOverNATS(js, "anchor")
	defer machine.Close()
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			took <- work.Request().Repository
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	var order []string
	for len(order) < 3 {
		select {
		case r := <-took:
			order = append(order, r)
		case <-time.After(10 * time.Second):
			t.Fatalf("taken so far: %v", order)
		}
	}
	if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
		t.Fatalf("taken as %v, want %v", order, want)
	}
}

// --- the moves, against a real server ------------------------------------------------------------

func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
	t.Helper()
	body, _ := json.Marshal(r)
	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
		t.Fatal(err)
	}
}

func aCheck(asked time.Time, repository string) BuildRequest {
	return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
}

func aDelivery(asked time.Time, repository string) BuildRequest {
	return BuildRequest{ID: NewBuildID(asked), Repository: repository}
}

// queued is the stream's asks, in queue order.
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
	t.Helper()
	q, err := ReadQueue(context.Background(), js, TheBuildMachine)
	if err != nil {
		t.Fatal(err)
	}
	return q.Asks
}

// takeAll runs a holder that records how often it was handed each repository.
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
	t.Helper()
	took := make(chan string, 100)
	var mu sync.Mutex
	counts := map[string]int{}
	machine := MachineOverNATS(js, "anchor")
	t.Cleanup(machine.Close)
	go func() {
		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
			mu.Lock()
			counts[work.Request().Repository]++
			mu.Unlock()
			took <- work.Request().Repository
			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
			_ = work.Done()
		})
	}()
	return took, func() map[string]int {
		mu.Lock()
		defer mu.Unlock()
		out := map[string]int{}
		for k, v := range counts {
			out[k] = v
		}
		return out
	}
}

// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	now := time.Now()
	check := aCheck(now.Add(-2*time.Minute), "/c")
	publishAsk(t, js, check)
	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
	publishAsk(t, js, check) // the copy, published; the original never deleted
	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
		t.Fatal(err)
	}
	var got []string
	for _, a := range queued(t, js) {
		got = append(got, a.Repository)
	}
	if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
		t.Fatalf("after the pass the queue holds %v, want %v", got, want)
	}

	// And one a holder took before the copy was published: the copy goes, the holder's stays.
	taken := aCheck(now, "/taken")
	publishAsk(t, js, taken)
	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = sub.Unsubscribe() }()
	// Take everything waiting up to and including /taken, and keep /taken unsettled.
	for {
		msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
		if err != nil || len(msgs) == 0 {
			t.Fatalf("nothing to take: %v", err)
		}
		var r BuildRequest
		_ = json.Unmarshal(msgs[0].Data, &r)
		if r.Repository == "/taken" {
			break
		}
		_ = msgs[0].Ack()
	}
	publishAsk(t, js, taken)
	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
		t.Fatal(err)
	}
	n := 0
	for _, a := range queued(t, js) {
		if a.ID == taken.ID {
			n++
		}
	}
	if n != 1 {
		t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
	}
}

// Two passes at once: nothing is built twice, and nothing is lost.
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	took, counts := takeAll(t, ctx, js)
	now := time.Now()
	const asks = 24
	var passes sync.WaitGroup
	for p := 0; p < 2; p++ {
		passes.Add(1)
		go func() {
			defer passes.Done()
			for i := 0; i < 15; i++ {
				_, _ = PutInOrder(ctx, js, TheBuildMachine)
			}
		}()
	}
	for i := 0; i < asks; i++ {
		repository := fmt.Sprintf("/%02d", i)
		if i%3 == 2 {
			publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
		} else {
			publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
		}
	}
	passes.Wait()
	for got := 0; got < asks; got++ {
		select {
		case <-took:
		case <-time.After(15 * time.Second):
			t.Fatalf("taken %d of %d: %v", got, asks, counts())
		}
	}
	time.Sleep(time.Second)
	c := counts()
	for i := 0; i < asks; i++ {
		if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
			t.Errorf("/%02d was built %d times", i, n)
		}
	}
}

// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx, stop := context.WithCancel(t.Context())
	defer stop()
	now := time.Now()
	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
	publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
		t.Fatal(err)
	}
	took, _ := takeAll(t, ctx, js)
	var order []string
	for len(order) < 2 {
		select {
		case r := <-took:
			order = append(order, r)
		case <-time.After(10 * time.Second):
			t.Fatalf("taken so far: %v", order)
		}
	}
	if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
		t.Fatalf("taken as %v, want %v", order, want)
	}
}

// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
// there was nothing left to delete.
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	now := time.Now()
	publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
	read := queued(t, js)[0]
	publishAsk(t, js, aDelivery(now, "/d"))
	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
		t.Fatalf("moved %v: %v", moved, err)
	}
	removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
	if err != nil || removed == 0 || removed == read.Seq {
		t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
	}
	for _, a := range queued(t, js) {
		if a.ID == read.ID {
			t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
		}
	}
	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
		t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
	}
}

// An ask nothing can be read from keeps its place in time when it moves.
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
	js := aBusWithTheBuildRole(t)
	ctx := t.Context()
	if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
		t.Fatal(err)
	}
	first := queued(t, js)[0].Stored
	time.Sleep(50 * time.Millisecond)
	publishAsk(t, js, aDelivery(time.Now(), "/d"))
	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
		t.Fatalf("moved %v: %v", moved, err)
	}
	asks := queued(t, js)
	if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
		t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
	}
}