package link import ( "context" "encoding/json" "errors" "fmt" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" ) // Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against // fake queues and a fake bus. // fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from. type fakeQueue struct { mu sync.Mutex waiting []*nats.Msg fetched int } func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) { f.mu.Lock() defer f.mu.Unlock() f.fetched++ if len(f.waiting) == 0 { return nil, nats.ErrTimeout } m := f.waiting[0] f.waiting = f.waiting[1:] return []*nats.Msg{m}, nil } func (f *fakeQueue) fetches() int { f.mu.Lock() defer f.mu.Unlock() return f.fetched } // askedAgo is a request asked this long ago, as the id says, for a check of this pull request. func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg { id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()) body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}}) return &nats.Msg{Data: body} } func buildAsk() *nats.Msg { body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"}) return &nats.Msg{Data: body} } // fakeGate is the bus as the loop sees it for the hold. type fakeGate struct { mu sync.Mutex plans []string plansErr error letKey bool taken int kept int dropped int oldestFrom map[*takeQueue]*fakeQueue } func (g *fakeGate) Building(context.Context) ([]string, error) { g.mu.Lock() defer g.mu.Unlock() return g.plans, g.plansErr } // OldestWaiting reads the first waiting ask of the fake queue without taking it. func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) { f := g.oldestFrom[q] f.mu.Lock() defer f.mu.Unlock() if len(f.waiting) == 0 { return time.Time{}, false, nil } return askedAtOf(f.waiting[0].Data, ""), true, nil } func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) { g.mu.Lock() defer g.mu.Unlock() if g.letKey { return false, nil } g.letKey = true g.taken++ return true, nil } func (g *fakeGate) KeepLetThrough(context.Context, string) error { g.mu.Lock() defer g.mu.Unlock() g.kept++ return nil } func (g *fakeGate) DropLetThrough(context.Context) error { g.mu.Lock() defer g.mu.Unlock() g.letKey = false g.dropped++ return nil } func (g *fakeGate) setPlans(plans ...string) { g.mu.Lock() defer g.mu.Unlock() g.plans = plans } // aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake. func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) { builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{} subs := []*takeQueue{ {kind: queueBuilds, pull: builds, bound: true}, {kind: queueReady, pull: ready, bound: true}, {kind: queueOther, pull: other, bound: true}, } gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}} m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate} return m, builds, ready, other, gate } func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) { t.Helper() msg, q, err := m.next(context.Background()) if err != nil { t.Fatal(err) } return msg, q } // 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment // (between two tiers). func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) { m, builds, ready, other, gate := aMachine() gate.setPlans("plan-1") ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)} other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)} // The delivery's queue is empty: between two tiers. if msg, _ := nextAsk(t, m); msg != nil { t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data) } if ready.fetches() != 0 || other.fetches() != 0 { t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches()) } if builds.fetches() == 0 { t.Fatal("the delivery's queue was not looked at") } } // A delivery build is still taken first, and the check waits behind it. func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) { m, builds, ready, _, gate := aMachine() gate.setPlans("plan-1") builds.waiting = []*nats.Msg{buildAsk()} ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)} msg, q := nextAsk(t, m) if msg == nil || q.kind != queueBuilds { t.Fatalf("the delivery build was not taken first: %v %v", msg, q) } if ready.fetches() != 0 { t.Fatal("a check queue was fetched from") } } // It is taken once the key is gone (deleted at the phase's end, or expired). func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) { m, _, ready, _, gate := aMachine() gate.setPlans("plan-1") ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)} if msg, _ := nextAsk(t, m); msg != nil { t.Fatal("taken while the key is there") } gate.setPlans() msg, q := nextAsk(t, m) if msg == nil || q.kind != queueReady { t.Fatalf("the check was not taken once the key was gone: %v", q) } if m.letKey { t.Fatal("a check taken without a delivery building took the let-through key") } } // 1687: a bucket that cannot be read is said and read as empty, so the checks are taken. func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) { m, _, ready, _, gate := aMachine() gate.plansErr = errors.New("permissions violation") gate.plans = []string{"plan-1"} ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)} msg, q := nextAsk(t, m) if msg == nil || q.kind != queueReady { t.Fatal("a check was held because the bucket could not be read") } if !m.said["the bucket of the plans building cannot be read"] { t.Fatalf("the fault was not said: %v", m.said) } } // 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a // waiting delivery build; a younger one is never taken before it. func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) { m, builds, ready, other, gate := aMachine() gate.setPlans("plan-1") builds.waiting = []*nats.Msg{buildAsk()} ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)} other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)} msg, q := nextAsk(t, m) if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) { t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q) } if !m.letKey || gate.taken != 1 { t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken) } } // 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds. func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) { m, builds, ready, other, gate := aMachine() gate.setPlans("plan-1") ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)} other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)} var order []string for range 3 { msg, q := nextAsk(t, m) if msg == nil { t.Fatalf("no check let through after %v", order) } var r BuildRequest _ = json.Unmarshal(msg.Data, &r) order = append(order, fmt.Sprint(r.Check.Number, q.kind)) // While it runs another holder (the same fake gate, key held) takes builds only. builds.waiting = []*nats.Msg{buildAsk()} other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate} if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds { t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq) } m.dropLetThrough(context.Background()) } if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") { t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want) } } // 1688: a check younger than the bound is never let through, and no key is created for it. func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) { m, _, ready, _, gate := aMachine() gate.setPlans("plan-1") ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)} if msg, _ := nextAsk(t, m); msg != nil { t.Fatal("a check of 29 minutes was let through") } if gate.taken != 0 { t.Fatal("a let-through key was created for a young check") } if ready.fetches() != 0 { t.Fatal("the age was read by a fetch; it must be read without taking") } } // 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the // moment between the read and the fetch is handed back with the key released. func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) { m, _, ready, _, gate := aMachine() gate.setPlans("plan-1") // The read sees the old check; before the fetch another holder takes it and a young one is first. gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}} ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)} if msg, _ := nextAsk(t, m); msg != nil { t.Fatalf("a young check was run during a delivery: %s", msg.Data) } if gate.letKey || m.letKey { t.Fatal("the let-through key was left held") } } // The key of a check let through is refreshed while it runs and deleted when it ends. func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) { m, _, ready, _, gate := aMachine() gate.setPlans("plan-1") ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)} if msg, _ := nextAsk(t, m); msg == nil { t.Fatal("not let through") } end := m.keepLetThrough(context.Background(), 20*time.Millisecond) time.Sleep(120 * time.Millisecond) gate.mu.Lock() held, kept := gate.letKey, gate.kept gate.mu.Unlock() if !held || kept == 0 { t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept) } end() gate.mu.Lock() defer gate.mu.Unlock() if gate.letKey || m.letKey { t.Error("the key was not deleted when the check ended") } }