jschoubben
a14dc04b7b
ADR 0483: the controller keeps a key per plan in its build phase; holders read it and take no new check while one is there, but for the oldest check past 30 minutes, one at a time by a let-through key. The queue says why checks wait.
308 lines
10 KiB
Go
308 lines
10 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
)
|
|
|
|
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
|
|
// fake queues and a fake bus.
|
|
|
|
// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
|
|
type fakeQueue struct {
|
|
mu sync.Mutex
|
|
waiting []*nats.Msg
|
|
fetched int
|
|
}
|
|
|
|
func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.fetched++
|
|
if len(f.waiting) == 0 {
|
|
return nil, nats.ErrTimeout
|
|
}
|
|
m := f.waiting[0]
|
|
f.waiting = f.waiting[1:]
|
|
return []*nats.Msg{m}, nil
|
|
}
|
|
|
|
func (f *fakeQueue) fetches() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.fetched
|
|
}
|
|
|
|
// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
|
|
func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
|
|
id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
|
|
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
|
|
return &nats.Msg{Data: body}
|
|
}
|
|
|
|
func buildAsk() *nats.Msg {
|
|
body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
|
|
return &nats.Msg{Data: body}
|
|
}
|
|
|
|
// fakeGate is the bus as the loop sees it for the hold.
|
|
type fakeGate struct {
|
|
mu sync.Mutex
|
|
plans []string
|
|
plansErr error
|
|
letKey bool
|
|
taken int
|
|
kept int
|
|
dropped int
|
|
oldestFrom map[*takeQueue]*fakeQueue
|
|
}
|
|
|
|
func (g *fakeGate) Building(context.Context) ([]string, error) {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
return g.plans, g.plansErr
|
|
}
|
|
|
|
// OldestWaiting reads the first waiting ask of the fake queue without taking it.
|
|
func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
|
|
f := g.oldestFrom[q]
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if len(f.waiting) == 0 {
|
|
return time.Time{}, false, nil
|
|
}
|
|
return askedAtOf(f.waiting[0].Data, ""), true, nil
|
|
}
|
|
|
|
func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
if g.letKey {
|
|
return false, nil
|
|
}
|
|
g.letKey = true
|
|
g.taken++
|
|
return true, nil
|
|
}
|
|
|
|
func (g *fakeGate) KeepLetThrough(context.Context, string) error {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
g.kept++
|
|
return nil
|
|
}
|
|
|
|
func (g *fakeGate) DropLetThrough(context.Context) error {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
g.letKey = false
|
|
g.dropped++
|
|
return nil
|
|
}
|
|
|
|
func (g *fakeGate) setPlans(plans ...string) {
|
|
g.mu.Lock()
|
|
defer g.mu.Unlock()
|
|
g.plans = plans
|
|
}
|
|
|
|
// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
|
|
func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
|
|
builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
|
|
subs := []*takeQueue{
|
|
{kind: queueBuilds, pull: builds, bound: true},
|
|
{kind: queueReady, pull: ready, bound: true},
|
|
{kind: queueOther, pull: other, bound: true},
|
|
}
|
|
gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
|
|
m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
|
|
return m, builds, ready, other, gate
|
|
}
|
|
|
|
func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
|
|
t.Helper()
|
|
msg, q, err := m.next(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return msg, q
|
|
}
|
|
|
|
// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
|
|
// (between two tiers).
|
|
func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
|
|
m, builds, ready, other, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
|
other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
|
|
// The delivery's queue is empty: between two tiers.
|
|
if msg, _ := nextAsk(t, m); msg != nil {
|
|
t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
|
|
}
|
|
if ready.fetches() != 0 || other.fetches() != 0 {
|
|
t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
|
|
}
|
|
if builds.fetches() == 0 {
|
|
t.Fatal("the delivery's queue was not looked at")
|
|
}
|
|
}
|
|
|
|
// A delivery build is still taken first, and the check waits behind it.
|
|
func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
|
|
m, builds, ready, _, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
builds.waiting = []*nats.Msg{buildAsk()}
|
|
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
|
msg, q := nextAsk(t, m)
|
|
if msg == nil || q.kind != queueBuilds {
|
|
t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
|
|
}
|
|
if ready.fetches() != 0 {
|
|
t.Fatal("a check queue was fetched from")
|
|
}
|
|
}
|
|
|
|
// It is taken once the key is gone (deleted at the phase's end, or expired).
|
|
func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
|
|
m, _, ready, _, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
|
if msg, _ := nextAsk(t, m); msg != nil {
|
|
t.Fatal("taken while the key is there")
|
|
}
|
|
gate.setPlans()
|
|
msg, q := nextAsk(t, m)
|
|
if msg == nil || q.kind != queueReady {
|
|
t.Fatalf("the check was not taken once the key was gone: %v", q)
|
|
}
|
|
if m.letKey {
|
|
t.Fatal("a check taken without a delivery building took the let-through key")
|
|
}
|
|
}
|
|
|
|
// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
|
|
func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
|
|
m, _, ready, _, gate := aMachine()
|
|
gate.plansErr = errors.New("permissions violation")
|
|
gate.plans = []string{"plan-1"}
|
|
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
|
msg, q := nextAsk(t, m)
|
|
if msg == nil || q.kind != queueReady {
|
|
t.Fatal("a check was held because the bucket could not be read")
|
|
}
|
|
if !m.said["the bucket of the plans building cannot be read"] {
|
|
t.Fatalf("the fault was not said: %v", m.said)
|
|
}
|
|
}
|
|
|
|
// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
|
|
// waiting delivery build; a younger one is never taken before it.
|
|
func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
|
|
m, builds, ready, other, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
builds.waiting = []*nats.Msg{buildAsk()}
|
|
ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
|
|
other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
|
|
msg, q := nextAsk(t, m)
|
|
if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
|
|
t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
|
|
}
|
|
if !m.letKey || gate.taken != 1 {
|
|
t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
|
|
}
|
|
}
|
|
|
|
// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
|
|
func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
|
|
m, builds, ready, other, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
|
|
other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
|
|
var order []string
|
|
for range 3 {
|
|
msg, q := nextAsk(t, m)
|
|
if msg == nil {
|
|
t.Fatalf("no check let through after %v", order)
|
|
}
|
|
var r BuildRequest
|
|
_ = json.Unmarshal(msg.Data, &r)
|
|
order = append(order, fmt.Sprint(r.Check.Number, q.kind))
|
|
// While it runs another holder (the same fake gate, key held) takes builds only.
|
|
builds.waiting = []*nats.Msg{buildAsk()}
|
|
other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
|
|
if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
|
|
t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
|
|
}
|
|
m.dropLetThrough(context.Background())
|
|
}
|
|
if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
|
|
t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
|
|
}
|
|
}
|
|
|
|
// 1688: a check younger than the bound is never let through, and no key is created for it.
|
|
func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
|
|
m, _, ready, _, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
|
|
if msg, _ := nextAsk(t, m); msg != nil {
|
|
t.Fatal("a check of 29 minutes was let through")
|
|
}
|
|
if gate.taken != 0 {
|
|
t.Fatal("a let-through key was created for a young check")
|
|
}
|
|
if ready.fetches() != 0 {
|
|
t.Fatal("the age was read by a fetch; it must be read without taking")
|
|
}
|
|
}
|
|
|
|
// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
|
|
// moment between the read and the fetch is handed back with the key released.
|
|
func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
|
|
m, _, ready, _, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
// The read sees the old check; before the fetch another holder takes it and a young one is first.
|
|
gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
|
|
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
|
if msg, _ := nextAsk(t, m); msg != nil {
|
|
t.Fatalf("a young check was run during a delivery: %s", msg.Data)
|
|
}
|
|
if gate.letKey || m.letKey {
|
|
t.Fatal("the let-through key was left held")
|
|
}
|
|
}
|
|
|
|
// The key of a check let through is refreshed while it runs and deleted when it ends.
|
|
func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
|
|
m, _, ready, _, gate := aMachine()
|
|
gate.setPlans("plan-1")
|
|
ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
|
|
if msg, _ := nextAsk(t, m); msg == nil {
|
|
t.Fatal("not let through")
|
|
}
|
|
end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
|
|
time.Sleep(120 * time.Millisecond)
|
|
gate.mu.Lock()
|
|
held, kept := gate.letKey, gate.kept
|
|
gate.mu.Unlock()
|
|
if !held || kept == 0 {
|
|
t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
|
|
}
|
|
end()
|
|
gate.mu.Lock()
|
|
defer gate.mu.Unlock()
|
|
if gate.letKey || m.letKey {
|
|
t.Error("the key was not deleted when the check ended")
|
|
}
|
|
}
|