jschoubben
fc573f951a
mesh/merge-gate head fc573f951a47 merged into trunk 2f157148c6c8 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head fc573f951a47 merged into trunk 2f157148c6c8 — pass: its merge-check.sh passed
mesh/delivery delivered
Second review of #276: a dead letter delivered again beside its original shares its request id and was removed as a copy, losing the ask; and a publish that outlived the pause could delete an original a holder had taken. A copy now names the message it copies, removals are said, the publish waits inside the pause's margin, and an original handed out meanwhile keeps its place while the copy goes.
442 lines
15 KiB
Go
442 lines
15 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
)
|
|
|
|
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
|
|
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
|
|
// first come, first served within a kind.
|
|
|
|
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
|
|
r := BuildRequest{ID: id, Repository: "/r"}
|
|
if check {
|
|
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
|
|
}
|
|
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
|
|
}
|
|
|
|
func idsOf(asks []QueuedAsk) []string {
|
|
out := []string{}
|
|
for _, a := range asks {
|
|
out = append(out, a.ID)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
|
|
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
waiting := []QueuedAsk{
|
|
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
|
|
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
|
|
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
|
|
}
|
|
want := []string{"delivery-d", "check-a", "check-b"}
|
|
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("handed out as %v, want %v", got, want)
|
|
}
|
|
// Only the checks ahead of it move behind it; the delivery stays where it is.
|
|
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
|
|
t.Fatalf("moved %v", got)
|
|
}
|
|
}
|
|
|
|
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
|
|
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
|
|
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
|
|
waiting := []QueuedAsk{
|
|
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
|
|
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
|
|
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
|
|
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
|
|
}
|
|
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
|
|
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("handed out as %v, want %v", got, want)
|
|
}
|
|
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
|
|
t.Fatalf("moved %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
|
|
for _, r := range []BuildRequest{
|
|
{}, // unreadable: nothing was read from it
|
|
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
|
|
{ID: "build-2", DryRun: true}, // looked at, never delivered
|
|
} {
|
|
if got := KindOf(r); got != KindCheck {
|
|
t.Errorf("%+v is a %s", r, got)
|
|
}
|
|
}
|
|
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
|
|
t.Errorf("a build is a %s", got)
|
|
}
|
|
}
|
|
|
|
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
|
|
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx, stop := context.WithCancel(context.Background())
|
|
defer stop()
|
|
asks := BuildsOn(js, TheBuildMachine)
|
|
now := time.Now()
|
|
for i, r := range []BuildRequest{
|
|
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
|
|
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
|
|
{ID: NewBuildID(now), Repository: "/d"},
|
|
} {
|
|
if i == 2 {
|
|
// And one nobody can read, whose kind cannot be told: it waits with the checks.
|
|
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := asks.Ask(ctx, r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// The acting controller's loop puts it in order, told by the asks.
|
|
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
|
|
func(string, ...any) {})
|
|
time.Sleep(500 * time.Millisecond)
|
|
took := make(chan string, 4)
|
|
machine := MachineOverNATS(js, "anchor")
|
|
defer machine.Close()
|
|
go func() {
|
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
|
took <- work.Request().Repository
|
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
|
_ = work.Done()
|
|
})
|
|
}()
|
|
var order []string
|
|
for len(order) < 3 {
|
|
select {
|
|
case r := <-took:
|
|
order = append(order, r)
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("taken so far: %v", order)
|
|
}
|
|
}
|
|
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
|
|
t.Fatalf("taken as %v, want %v", order, want)
|
|
}
|
|
}
|
|
|
|
// --- the moves, against a real server ------------------------------------------------------------
|
|
|
|
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(r)
|
|
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func aCheck(asked time.Time, repository string) BuildRequest {
|
|
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
|
|
}
|
|
|
|
func aDelivery(asked time.Time, repository string) BuildRequest {
|
|
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
|
|
}
|
|
|
|
// queued is the stream's asks, in queue order.
|
|
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
|
|
t.Helper()
|
|
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return q.Asks
|
|
}
|
|
|
|
// takeAll runs a holder that records how often it was handed each repository.
|
|
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
|
|
t.Helper()
|
|
took := make(chan string, 100)
|
|
var mu sync.Mutex
|
|
counts := map[string]int{}
|
|
machine := MachineOverNATS(js, "anchor")
|
|
t.Cleanup(machine.Close)
|
|
go func() {
|
|
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
|
mu.Lock()
|
|
counts[work.Request().Repository]++
|
|
mu.Unlock()
|
|
took <- work.Request().Repository
|
|
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
|
_ = work.Done()
|
|
})
|
|
}()
|
|
return took, func() map[string]int {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
out := map[string]int{}
|
|
for k, v := range counts {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
}
|
|
|
|
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
|
|
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
|
|
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx := t.Context()
|
|
now := time.Now()
|
|
check := aCheck(now.Add(-2*time.Minute), "/c")
|
|
publishAsk(t, js, check)
|
|
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
|
|
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
|
|
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
|
|
t.Fatalf("removed %v: %v", pass.Removed, err)
|
|
}
|
|
var got []string
|
|
for _, a := range queued(t, js) {
|
|
got = append(got, a.Repository)
|
|
}
|
|
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
|
|
}
|
|
|
|
// And one a holder took before the copy was published: the copy goes, the holder's stays.
|
|
taken := aCheck(now, "/taken")
|
|
publishAsk(t, js, taken)
|
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = sub.Unsubscribe() }()
|
|
// Take everything waiting up to and including /taken, and keep /taken unsettled.
|
|
for {
|
|
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
|
if err != nil || len(msgs) == 0 {
|
|
t.Fatalf("nothing to take: %v", err)
|
|
}
|
|
var r BuildRequest
|
|
_ = json.Unmarshal(msgs[0].Data, &r)
|
|
if r.Repository == "/taken" {
|
|
break
|
|
}
|
|
_ = msgs[0].Ack()
|
|
}
|
|
moveCopyOf(t, js, taken.ID)
|
|
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n := 0
|
|
for _, a := range queued(t, js) {
|
|
if a.ID == taken.ID {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
|
|
}
|
|
}
|
|
|
|
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
|
|
// would leave it.
|
|
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
|
|
t.Helper()
|
|
for _, a := range queued(t, js) {
|
|
if a.ID != id {
|
|
continue
|
|
}
|
|
body, _ := json.Marshal(a.Request)
|
|
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
|
|
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
|
|
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
|
|
if _, err := js.Context().PublishMsg(msg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return
|
|
}
|
|
t.Fatalf("%s is not queued", id)
|
|
}
|
|
|
|
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
|
|
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
|
|
// copy, whose headers the delivery carries.
|
|
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx := t.Context()
|
|
ask := aCheck(time.Now(), "/again")
|
|
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
|
|
publishAsk(t, js, ask)
|
|
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
|
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
|
|
t.Fatal(err)
|
|
}
|
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
|
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = sub.Unsubscribe() }()
|
|
var original *nats.Msg
|
|
for original == nil {
|
|
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
|
if err != nil || len(msgs) == 0 {
|
|
t.Fatalf("nothing to take: %v", err)
|
|
}
|
|
var r BuildRequest
|
|
_ = json.Unmarshal(msgs[0].Data, &r)
|
|
if r.ID == ask.ID {
|
|
original = msgs[0] // given up on, never settled: dead, and kept
|
|
continue
|
|
}
|
|
_ = msgs[0].Ack()
|
|
}
|
|
// Delivered again as DeliverAgain does: its headers, and its own.
|
|
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
|
|
for k, v := range original.Header {
|
|
again.Header[k] = v
|
|
}
|
|
again.Header.Set(AgainHeader, "7")
|
|
if _, err := js.Context().PublishMsg(again); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pass, err := PutInOrder(ctx, js, TheBuildMachine)
|
|
if err != nil || len(pass.Removed) != 0 {
|
|
t.Fatalf("removed %v: %v", pass.Removed, err)
|
|
}
|
|
n := 0
|
|
for _, a := range queued(t, js) {
|
|
if a.ID == ask.ID {
|
|
n++
|
|
}
|
|
}
|
|
if n != 2 {
|
|
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
|
|
}
|
|
}
|
|
|
|
// Two passes at once: nothing is built twice, and nothing is lost.
|
|
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx, stop := context.WithCancel(t.Context())
|
|
defer stop()
|
|
took, counts := takeAll(t, ctx, js)
|
|
now := time.Now()
|
|
const asks = 24
|
|
var passes sync.WaitGroup
|
|
for p := 0; p < 2; p++ {
|
|
passes.Add(1)
|
|
go func() {
|
|
defer passes.Done()
|
|
for i := 0; i < 15; i++ {
|
|
_, _ = PutInOrder(ctx, js, TheBuildMachine)
|
|
}
|
|
}()
|
|
}
|
|
for i := 0; i < asks; i++ {
|
|
repository := fmt.Sprintf("/%02d", i)
|
|
if i%3 == 2 {
|
|
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
|
|
} else {
|
|
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
|
|
}
|
|
}
|
|
passes.Wait()
|
|
for got := 0; got < asks; got++ {
|
|
select {
|
|
case <-took:
|
|
case <-time.After(15 * time.Second):
|
|
t.Fatalf("taken %d of %d: %v", got, asks, counts())
|
|
}
|
|
}
|
|
time.Sleep(time.Second)
|
|
c := counts()
|
|
for i := 0; i < asks; i++ {
|
|
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
|
|
t.Errorf("/%02d was built %d times", i, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
|
|
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx, stop := context.WithCancel(t.Context())
|
|
defer stop()
|
|
now := time.Now()
|
|
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
|
|
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
|
|
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
took, _ := takeAll(t, ctx, js)
|
|
var order []string
|
|
for len(order) < 2 {
|
|
select {
|
|
case r := <-took:
|
|
order = append(order, r)
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("taken so far: %v", order)
|
|
}
|
|
}
|
|
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
|
|
t.Fatalf("taken as %v, want %v", order, want)
|
|
}
|
|
}
|
|
|
|
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
|
|
// there was nothing left to delete.
|
|
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx := t.Context()
|
|
now := time.Now()
|
|
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
|
|
read := queued(t, js)[0]
|
|
publishAsk(t, js, aDelivery(now, "/d"))
|
|
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
|
t.Fatalf("moved %v: %v", pass.Moved, err)
|
|
}
|
|
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
|
|
if err != nil || removed == 0 || removed == read.Seq {
|
|
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
|
|
}
|
|
for _, a := range queued(t, js) {
|
|
if a.ID == read.ID {
|
|
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
|
|
}
|
|
}
|
|
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
|
|
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
|
|
}
|
|
}
|
|
|
|
// An ask nothing can be read from keeps its place in time when it moves.
|
|
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
|
|
js := aBusWithTheBuildRole(t)
|
|
ctx := t.Context()
|
|
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := queued(t, js)[0].Stored
|
|
time.Sleep(50 * time.Millisecond)
|
|
publishAsk(t, js, aDelivery(time.Now(), "/d"))
|
|
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
|
|
t.Fatalf("moved %v: %v", pass.Moved, err)
|
|
}
|
|
asks := queued(t, js)
|
|
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
|
|
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
|
|
}
|
|
}
|