package link import ( "context" "encoding/json" "fmt" "reflect" "sync" "testing" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" ) // The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any // pull request's check, a check that has waited ten minutes before every delivery asked after it, and // first come, first served within a kind. func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk { r := BuildRequest{ID: id, Repository: "/r"} if check { r.Check = &CheckRequest{Owner: "novox", Repo: "r"} } return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting} } func idsOf(asks []QueuedAsk) []string { out := []string{} for _, a := range asks { out = append(out, a.ID) } return out } func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) { now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) waiting := []QueuedAsk{ anAsk(1, "check-a", now.Add(-3*time.Minute), true), anAsk(2, "check-b", now.Add(-2*time.Minute), true), anAsk(3, "delivery-d", now.Add(-time.Minute), false), } want := []string{"delivery-d", "check-a", "check-b"} if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { t.Fatalf("handed out as %v, want %v", got, want) } // Only the checks ahead of it move behind it; the delivery stays where it is. if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) { t.Fatalf("moved %v", got) } } func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) { now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) // As the queue stands after a delivery put the young checks behind it, ten minutes ago and since. waiting := []QueuedAsk{ anAsk(5, "delivery-early", now.Add(-12*time.Minute), false), anAsk(6, "delivery-later", now.Add(-time.Minute), false), anAsk(7, "check-old", now.Add(-11*time.Minute), true), anAsk(8, "check-young", now.Add(-9*time.Minute), true), } want := []string{"delivery-early", "check-old", "delivery-later", "check-young"} if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { t.Fatalf("handed out as %v, want %v", got, want) } if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) { t.Fatalf("moved %v", got) } } func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) { for _, r := range []BuildRequest{ {}, // unreadable: nothing was read from it {ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked {ID: "build-2", DryRun: true}, // looked at, never delivered } { if got := KindOf(r); got != KindCheck { t.Errorf("%+v is a %s", r, got) } } if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery { t.Errorf("a build is a %s", got) } } // Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery. func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) { js := aBusWithTheBuildRole(t) ctx, stop := context.WithCancel(context.Background()) defer stop() asks := BuildsOn(js, TheBuildMachine) now := time.Now() for i, r := range []BuildRequest{ {ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}}, {ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}}, {ID: NewBuildID(now), Repository: "/d"}, } { if i == 2 { // And one nobody can read, whose kind cannot be told: it waits with the checks. if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil { t.Fatal(err) } } if err := asks.Ask(ctx, r); err != nil { t.Fatal(err) } } // The acting controller's loop puts it in order, told by the asks. go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true }, func(string, ...any) {}) time.Sleep(500 * time.Millisecond) took := make(chan string, 4) machine := MachineOverNATS(js, "anchor") defer machine.Close() go func() { _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().Repository _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) _ = work.Done() }) }() var order []string for len(order) < 3 { select { case r := <-took: order = append(order, r) case <-time.After(10 * time.Second): t.Fatalf("taken so far: %v", order) } } if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) { t.Fatalf("taken as %v, want %v", order, want) } } // --- the moves, against a real server ------------------------------------------------------------ func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) { t.Helper() body, _ := json.Marshal(r) if _, err := js.Context().Publish(BuildWork(), body); err != nil { t.Fatal(err) } } func aCheck(asked time.Time, repository string) BuildRequest { return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}} } func aDelivery(asked time.Time, repository string) BuildRequest { return BuildRequest{ID: NewBuildID(asked), Repository: repository} } // queued is the stream's asks, in queue order. func queued(t *testing.T, js *broker.JetStream) []QueuedAsk { t.Helper() q, err := ReadQueue(context.Background(), js, TheBuildMachine) if err != nil { t.Fatal(err) } return q.Asks } // takeAll runs a holder that records how often it was handed each repository. func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) { t.Helper() took := make(chan string, 100) var mu sync.Mutex counts := map[string]int{} machine := MachineOverNATS(js, "anchor") t.Cleanup(machine.Close) go func() { _ = machine.Take(ctx, func(ctx context.Context, work Build) { mu.Lock() counts[work.Request().Repository]++ mu.Unlock() took <- work.Request().Repository _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) _ = work.Done() }) }() return took, func() map[string]int { mu.Lock() defer mu.Unlock() out := map[string]int{} for k, v := range counts { out[k] = v } return out } } // A move that published its copy and stopped before deleting the original leaves two copies of one ask; // the next pass makes them one, and a copy of one a holder already took is removed, never built twice. func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) { js := aBusWithTheBuildRole(t) ctx := t.Context() now := time.Now() check := aCheck(now.Add(-2*time.Minute), "/c") publishAsk(t, js, check) publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d")) moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 { t.Fatalf("removed %v: %v", pass.Removed, err) } var got []string for _, a := range queued(t, js) { got = append(got, a.Repository) } if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) { t.Fatalf("after the pass the queue holds %v, want %v", got, want) } // And one a holder took before the copy was published: the copy goes, the holder's stays. taken := aCheck(now, "/taken") publishAsk(t, js, taken) worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}) sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) if err != nil { t.Fatal(err) } defer func() { _ = sub.Unsubscribe() }() // Take everything waiting up to and including /taken, and keep /taken unsettled. for { msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second)) if err != nil || len(msgs) == 0 { t.Fatalf("nothing to take: %v", err) } var r BuildRequest _ = json.Unmarshal(msgs[0].Data, &r) if r.Repository == "/taken" { break } _ = msgs[0].Ack() } moveCopyOf(t, js, taken.ID) if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { t.Fatal(err) } n := 0 for _, a := range queued(t, js) { if a.ID == taken.ID { n++ } } if n != 1 { t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n) } } // moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete // would leave it. func moveCopyOf(t *testing.T, js *broker.JetStream, id string) { t.Helper() for _, a := range queued(t, js) { if a.ID != id { continue } body, _ := json.Marshal(a.Request) msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}} msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano)) msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq)) if _, err := js.Context().PublishMsg(msg); err != nil { t.Fatal(err) } return } t.Fatalf("%s is not queued", id) } // A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real // ask with the same id, and is never removed as a second copy — even when the original was itself a move's // copy, whose headers the delivery carries. func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) { js := aBusWithTheBuildRole(t) ctx := t.Context() ask := aCheck(time.Now(), "/again") publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before")) publishAsk(t, js, ask) publishAsk(t, js, aDelivery(time.Now(), "/d")) if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy t.Fatal(err) } worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}) sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) if err != nil { t.Fatal(err) } defer func() { _ = sub.Unsubscribe() }() var original *nats.Msg for original == nil { msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second)) if err != nil || len(msgs) == 0 { t.Fatalf("nothing to take: %v", err) } var r BuildRequest _ = json.Unmarshal(msgs[0].Data, &r) if r.ID == ask.ID { original = msgs[0] // given up on, never settled: dead, and kept continue } _ = msgs[0].Ack() } // Delivered again as DeliverAgain does: its headers, and its own. again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}} for k, v := range original.Header { again.Header[k] = v } again.Header.Set(AgainHeader, "7") if _, err := js.Context().PublishMsg(again); err != nil { t.Fatal(err) } pass, err := PutInOrder(ctx, js, TheBuildMachine) if err != nil || len(pass.Removed) != 0 { t.Fatalf("removed %v: %v", pass.Removed, err) } n := 0 for _, a := range queued(t, js) { if a.ID == ask.ID { n++ } } if n != 2 { t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n) } } // Two passes at once: nothing is built twice, and nothing is lost. func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) { js := aBusWithTheBuildRole(t) ctx, stop := context.WithCancel(t.Context()) defer stop() took, counts := takeAll(t, ctx, js) now := time.Now() const asks = 24 var passes sync.WaitGroup for p := 0; p < 2; p++ { passes.Add(1) go func() { defer passes.Done() for i := 0; i < 15; i++ { _, _ = PutInOrder(ctx, js, TheBuildMachine) } }() } for i := 0; i < asks; i++ { repository := fmt.Sprintf("/%02d", i) if i%3 == 2 { publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository)) } else { publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository)) } } passes.Wait() for got := 0; got < asks; got++ { select { case <-took: case <-time.After(15 * time.Second): t.Fatalf("taken %d of %d: %v", got, asks, counts()) } } time.Sleep(time.Second) c := counts() for i := 0; i < asks; i++ { if n := c[fmt.Sprintf("/%02d", i)]; n != 1 { t.Errorf("/%02d was built %d times", i, n) } } } // A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it. func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) { js := aBusWithTheBuildRole(t) ctx, stop := context.WithCancel(t.Context()) defer stop() now := time.Now() publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery")) publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check")) if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { t.Fatal(err) } took, _ := takeAll(t, ctx, js) var order []string for len(order) < 2 { select { case r := <-took: order = append(order, r) case <-time.After(10 * time.Second): t.Fatalf("taken so far: %v", order) } } if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) { t.Fatalf("taken as %v, want %v", order, want) } } // A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when // there was nothing left to delete. func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) { js := aBusWithTheBuildRole(t) ctx := t.Context() now := time.Now() publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c")) read := queued(t, js)[0] publishAsk(t, js, aDelivery(now, "/d")) if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 { t.Fatalf("moved %v: %v", pass.Moved, err) } removed, err := DeleteAsk(ctx, js, TheBuildMachine, read) if err != nil || removed == 0 || removed == read.Seq { t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err) } for _, a := range queued(t, js) { if a.ID == read.ID { t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq) } } if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 { t.Fatalf("an ask gone already is said removed at %d: %v", removed, err) } } // An ask nothing can be read from keeps its place in time when it moves. func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) { js := aBusWithTheBuildRole(t) ctx := t.Context() if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil { t.Fatal(err) } first := queued(t, js)[0].Stored time.Sleep(50 * time.Millisecond) publishAsk(t, js, aDelivery(time.Now(), "/d")) if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 { t.Fatalf("moved %v: %v", pass.Moved, err) } asks := queued(t, js) if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) { t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks) } }