Make a queue move lossless and run its passes on the acting controller alone (hq issue 558)

Review of #276: a move deleted before it published, so a failed publish
lost an ask; a verb's own process ran a pass beside the serving one, so
one could resume the queue while the other moved asks and a build ran
twice. The copy is now published first and second copies removed by id,
asks only signal the acting controller's loop, moves stop with 5 s of
the pause left, a cancel finds a moved ask by id, and an unreadable ask
keeps its age.

internal/link/queue_order_test.go

@@ -2,9 +2,16 @@ package link
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"reflect"
+	"sync"
 	"testing"
 	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
 )
 
 // The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
@@ -99,6 +106,10 @@ func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+	// The acting controller's loop puts it in order, told by the asks.
+	go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
+		func(string, ...any) {})
+	time.Sleep(500 * time.Millisecond)
 	took := make(chan string, 4)
 	machine := MachineOverNATS(js, "anchor")
 	defer machine.Close()
@@ -122,3 +133,232 @@ func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
 		t.Fatalf("taken as %v, want %v", order, want)
 	}
 }
+
+// --- the moves, against a real server ------------------------------------------------------------
+
+func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
+	t.Helper()
+	body, _ := json.Marshal(r)
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func aCheck(asked time.Time, repository string) BuildRequest {
+	return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
+}
+
+func aDelivery(asked time.Time, repository string) BuildRequest {
+	return BuildRequest{ID: NewBuildID(asked), Repository: repository}
+}
+
+// queued is the stream's asks, in queue order.
+func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
+	t.Helper()
+	q, err := ReadQueue(context.Background(), js, TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return q.Asks
+}
+
+// takeAll runs a holder that records how often it was handed each repository.
+func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
+	t.Helper()
+	took := make(chan string, 100)
+	var mu sync.Mutex
+	counts := map[string]int{}
+	machine := MachineOverNATS(js, "anchor")
+	t.Cleanup(machine.Close)
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			mu.Lock()
+			counts[work.Request().Repository]++
+			mu.Unlock()
+			took <- work.Request().Repository
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	return took, func() map[string]int {
+		mu.Lock()
+		defer mu.Unlock()
+		out := map[string]int{}
+		for k, v := range counts {
+			out[k] = v
+		}
+		return out
+	}
+}
+
+// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
+// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
+func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	now := time.Now()
+	check := aCheck(now.Add(-2*time.Minute), "/c")
+	publishAsk(t, js, check)
+	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
+	publishAsk(t, js, check) // the copy, published; the original never deleted
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, a := range queued(t, js) {
+		got = append(got, a.Repository)
+	}
+	if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("after the pass the queue holds %v, want %v", got, want)
+	}
+
+	// And one a holder took before the copy was published: the copy goes, the holder's stays.
+	taken := aCheck(now, "/taken")
+	publishAsk(t, js, taken)
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	// Take everything waiting up to and including /taken, and keep /taken unsettled.
+	for {
+		msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
+		if err != nil || len(msgs) == 0 {
+			t.Fatalf("nothing to take: %v", err)
+		}
+		var r BuildRequest
+		_ = json.Unmarshal(msgs[0].Data, &r)
+		if r.Repository == "/taken" {
+			break
+		}
+		_ = msgs[0].Ack()
+	}
+	publishAsk(t, js, taken)
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
+		t.Fatal(err)
+	}
+	n := 0
+	for _, a := range queued(t, js) {
+		if a.ID == taken.ID {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
+	}
+}
+
+// Two passes at once: nothing is built twice, and nothing is lost.
+func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	took, counts := takeAll(t, ctx, js)
+	now := time.Now()
+	const asks = 24
+	var passes sync.WaitGroup
+	for p := 0; p < 2; p++ {
+		passes.Add(1)
+		go func() {
+			defer passes.Done()
+			for i := 0; i < 15; i++ {
+				_, _ = PutInOrder(ctx, js, TheBuildMachine)
+			}
+		}()
+	}
+	for i := 0; i < asks; i++ {
+		repository := fmt.Sprintf("/%02d", i)
+		if i%3 == 2 {
+			publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
+		} else {
+			publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
+		}
+	}
+	passes.Wait()
+	for got := 0; got < asks; got++ {
+		select {
+		case <-took:
+		case <-time.After(15 * time.Second):
+			t.Fatalf("taken %d of %d: %v", got, asks, counts())
+		}
+	}
+	time.Sleep(time.Second)
+	c := counts()
+	for i := 0; i < asks; i++ {
+		if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
+			t.Errorf("/%02d was built %d times", i, n)
+		}
+	}
+}
+
+// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
+func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	now := time.Now()
+	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
+	publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
+		t.Fatal(err)
+	}
+	took, _ := takeAll(t, ctx, js)
+	var order []string
+	for len(order) < 2 {
+		select {
+		case r := <-took:
+			order = append(order, r)
+		case <-time.After(10 * time.Second):
+			t.Fatalf("taken so far: %v", order)
+		}
+	}
+	if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("taken as %v, want %v", order, want)
+	}
+}
+
+// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
+// there was nothing left to delete.
+func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	now := time.Now()
+	publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
+	read := queued(t, js)[0]
+	publishAsk(t, js, aDelivery(now, "/d"))
+	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
+		t.Fatalf("moved %v: %v", moved, err)
+	}
+	removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
+	if err != nil || removed == 0 || removed == read.Seq {
+		t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
+	}
+	for _, a := range queued(t, js) {
+		if a.ID == read.ID {
+			t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
+		}
+	}
+	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
+		t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
+	}
+}
+
+// An ask nothing can be read from keeps its place in time when it moves.
+func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
+		t.Fatal(err)
+	}
+	first := queued(t, js)[0].Stored
+	time.Sleep(50 * time.Millisecond)
+	publishAsk(t, js, aDelivery(time.Now(), "/d"))
+	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
+		t.Fatalf("moved %v: %v", moved, err)
+	}
+	asks := queued(t, js)
+	if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
+		t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
+	}
+}