Hold the build seat's checks while a delivery builds, and let one check that waited 30 minutes through
ADR 0483: the controller keeps a key per plan in its build phase; holders read it and take no new check while one is there, but for the oldest check past 30 minutes, one at a time by a let-through key. The queue says why checks wait.
This commit is contained in:
+109
-64
@@ -162,11 +162,19 @@ type natsMachine struct {
|
||||
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
|
||||
subs []*takeQueue
|
||||
opts MachineOptions
|
||||
// gate is what the loop asks of the bus to hold checks while a delivery builds; nil is the bus's (a test sets a fake).
|
||||
gate checkGate
|
||||
// said is the faults of the hold already said, letKey whether this holder created the let-through key for the check it
|
||||
// took last (ADR 0483 clause 3).
|
||||
said map[string]bool
|
||||
letKey bool
|
||||
}
|
||||
|
||||
// takeQueue is one queue a machine pulls from.
|
||||
type takeQueue struct {
|
||||
sub *nats.Subscription
|
||||
// pull is what the queue is fetched through: the subscription once bound (a test sets a fake).
|
||||
pull puller
|
||||
// kind is what its asks are: a build, a ready check, or another check.
|
||||
kind string
|
||||
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
|
||||
@@ -174,6 +182,11 @@ type takeQueue struct {
|
||||
bound bool
|
||||
}
|
||||
|
||||
// puller is the fetch of a queue.
|
||||
type puller interface {
|
||||
Fetch(batch int, opts ...nats.PullOpt) ([]*nats.Msg, error)
|
||||
}
|
||||
|
||||
// The kinds of queue a machine takes from, in the order it takes from them.
|
||||
const (
|
||||
queueBuilds = "builds"
|
||||
@@ -195,6 +208,9 @@ type MachineOptions struct {
|
||||
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
|
||||
// the other, never neither. Nil claims every one.
|
||||
Claim func() bool
|
||||
// LetThroughAfter is how long a check waits while a delivery builds before it is let through; zero is ADR 0483's thirty
|
||||
// minutes. Set by a test.
|
||||
LetThroughAfter time.Duration
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -244,6 +260,9 @@ func (m *natsMachine) Close() {
|
||||
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
|
||||
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
|
||||
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
|
||||
//
|
||||
// **Unless a delivery builds** (novox/hq ADR 0483): while the controller's bucket of the plans building holds a key, no check queue
|
||||
// is fetched from, but for the one check that waited LetThroughAfter, which goes first (see next).
|
||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||
@@ -320,70 +339,79 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
|
||||
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
|
||||
// ask; running it a second time would spend a second place on one check.
|
||||
if why := m.runningElsewhere(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
m.handOver(ctx, msg, q, do)
|
||||
}
|
||||
}
|
||||
|
||||
// handOver settles one fetched ask: dropped, handed back or built. **A check let through keeps its key for as long as it
|
||||
// stays here** (ADR 0483 clause 3): refreshed while it runs and deleted when it ends, however it ends.
|
||||
func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue, do func(context.Context, Build)) {
|
||||
if m.letKey {
|
||||
defer m.keepLetThrough(ctx, heartbeatWhileWorking)()
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
return
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
|
||||
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
|
||||
// ask; running it a second time would spend a second place on one check.
|
||||
if why := m.runningElsewhere(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
}
|
||||
|
||||
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
|
||||
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
|
||||
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
|
||||
@@ -425,20 +453,32 @@ func (m *natsMachine) bind(q *takeQueue) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.sub, q.bound = sub, true
|
||||
q.sub, q.pull, q.bound = sub, sub, true
|
||||
return nil
|
||||
}
|
||||
|
||||
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
|
||||
// on for a second. Nothing, with no error, when there was nothing.
|
||||
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
// **While a delivery builds, only delivery builds are taken, but for one check that waited** (novox/hq ADR 0483): the
|
||||
// bucket of the plans building is read before either check queue is fetched from, and a key there holds both. The check
|
||||
// that waited LetThroughAfter goes first, ahead of the builds that wait; the rest of the holders take builds only.
|
||||
if m.seat == broker.TheBuildSeat && len(m.subs) > 1 {
|
||||
if plans := m.building(ctx); len(plans) > 0 {
|
||||
if msg, q := m.letOneThrough(ctx); msg != nil {
|
||||
fmt.Fprintf(os.Stderr, "a check that waited %s is let through while %s builds\n", m.letThroughAfter(), whyHeld(plans))
|
||||
return msg, q, nil
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
}
|
||||
var last error
|
||||
for _, q := range m.subs {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
fetched, err := q.sub.Fetch(1, nats.Context(trying))
|
||||
fetched, err := q.pull.Fetch(1, nats.Context(trying))
|
||||
end()
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], q, nil
|
||||
@@ -454,9 +494,14 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if last != nil {
|
||||
return nil, nil, last
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
|
||||
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
|
||||
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||
defer end()
|
||||
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
|
||||
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], m.subs[0], nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
//go:build integration
|
||||
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687, 1688), against a real bus.
|
||||
|
||||
// aBusWithTheBuckets is the bus with the build role and the controller's buckets, the bucket of the checks running with an age
|
||||
// of two seconds so a key a holder stopped refreshing is seen to expire.
|
||||
func aBusWithTheBuckets(t *testing.T) (*broker.JetStream, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
js := aBusWithTheBuildRole(t)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: 2 * time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plans, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.PlansBuildingBucket, History: 1, TTL: broker.PlansBuildingKeptFor})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, plans
|
||||
}
|
||||
|
||||
func aCheckAskedAgo(ago time.Duration, number int, ready bool) BuildRequest {
|
||||
r := aPullCheck(fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()), number, fmt.Sprintf("head-%d-0000000", number), "trunk-0-000000", ready)
|
||||
return r
|
||||
}
|
||||
|
||||
// 1652/AC6: a plan's builds asked in two tiers with a check waiting, the check is handed out only after the last tier's builds,
|
||||
// and not in the gap between the tiers.
|
||||
func TestNatsACheckIsHandedOutOnlyAfterTheLastTierOfAPlan(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(time.Minute, 7, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-1-a", Repository: "/a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took := make(chan string, 16)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
expect := func(want string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case got := <-took:
|
||||
if got != want {
|
||||
t.Fatalf("taken %s, want %s", got, want)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("%s was not taken", want)
|
||||
}
|
||||
}
|
||||
expect("tier-1-a")
|
||||
// The gap between two tiers: the delivery's queue is empty, the plan's key is there, and the check is not taken.
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken in the gap between two tiers", got)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-2-a", Repository: "/b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expect("tier-2-a")
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken before the plan's last build was built and its key deleted", got)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
// The phase ends: the key is deleted, and the check is handed out.
|
||||
if err := plans.Delete(ctx, "plan-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-took:
|
||||
if got == "tier-1-a" || got == "tier-2-a" {
|
||||
t.Fatalf("a build was taken twice: %s", got)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the check was not handed out once the plan's key was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: two holders racing for checks that waited past the bound: exactly one is let through at a time, by a create that fails
|
||||
// while the key is there, and the key is deleted when its check ends, after which the next oldest goes.
|
||||
func TestNatsTwoHoldersRacingForOldChecksLetExactlyOneThrough(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
// The buckets' age is two seconds here, so the keys are refreshed well within it.
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, ago := range []time.Duration{50 * time.Minute, 40 * time.Minute, 35 * time.Minute} {
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(ago, i+1, false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
started := make(chan string, 8)
|
||||
release := make(chan struct{})
|
||||
do := func(ctx context.Context, work Build) {
|
||||
mu.Lock()
|
||||
running++
|
||||
most = max(most, running)
|
||||
mu.Unlock()
|
||||
started <- work.Request().Check.Repo + fmt.Sprint(work.Request().Check.Number)
|
||||
<-release
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "x", Failed: "no"})
|
||||
_ = work.Done()
|
||||
}
|
||||
for _, on := range []string{"ace", "shanks"} {
|
||||
machine := MachineOverNATS(js, on)
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, do) }()
|
||||
}
|
||||
var order []string
|
||||
for range 3 {
|
||||
select {
|
||||
case id := <-started:
|
||||
order = append(order, id)
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("started so far: %v", order)
|
||||
}
|
||||
// While it runs, no second check starts and the key is there.
|
||||
select {
|
||||
case id := <-started:
|
||||
t.Fatalf("%s started beside a check let through", id)
|
||||
case <-time.After(2500 * time.Millisecond):
|
||||
}
|
||||
kv := runningKV(t, js)
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
t.Fatalf("the let-through key is not there while the check runs: %v", err)
|
||||
}
|
||||
release <- struct{}{}
|
||||
// And it is deleted when the check ends.
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the let-through key was not deleted when the check ended")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if want := []string{"a1", "a2", "a3"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("checks went %v, want the oldest first %v", order, want)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if most != 1 {
|
||||
t.Fatalf("%d checks ran at once while a delivery built", most)
|
||||
}
|
||||
}
|
||||
|
||||
func runningKV(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(t.Context(), broker.CheckRunningBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// 1688: the create of the let-through key succeeds for exactly one of holders asking at once, and a key its holder stopped
|
||||
// refreshing expires, after which the next create succeeds.
|
||||
func TestNatsOneCreateOfTheLetThroughKeySucceedsAndAStoppedHoldersKeyExpires(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
gate := busGate{conn: js.Conn()}
|
||||
ctx := t.Context()
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
won := 0
|
||||
for i := range 8 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ok, err := gate.TakeLetThrough(ctx, fmt.Sprint("holder-", i))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
mu.Lock()
|
||||
won++
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if won != 1 {
|
||||
t.Fatalf("%d of 8 holders created the let-through key, want exactly one", won)
|
||||
}
|
||||
// The holder stopped: nothing refreshes the key, and the bucket's age (two seconds here) removes it.
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
ok, err := gate.TakeLetThrough(ctx, "next")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the key of a stopped holder did not expire")
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
// And a refreshed key does not expire.
|
||||
for range 6 {
|
||||
if err := gate.KeepLetThrough(ctx, "next"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
}
|
||||
if ok, _ := gate.TakeLetThrough(ctx, "other"); ok {
|
||||
t.Fatal("a refreshed key expired")
|
||||
}
|
||||
var v LetThrough
|
||||
e, err := runningKV(t, js).Get(ctx, LetThroughKey)
|
||||
if err != nil || json.Unmarshal(e.Value(), &v) != nil || v.Builder != "next" {
|
||||
t.Fatalf("the key says %+v (%v)", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: the oldest waiting ask is read without being taken: no delivery is counted against it.
|
||||
func TestNatsTheOldestWaitingAskIsReadWithoutTakingIt(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
ctx := t.Context()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
old := aCheckAskedAgo(45*time.Minute, 1, false)
|
||||
if err := asks.Ask(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := busGate{conn: js.Conn()}
|
||||
q := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the first waiting ask was not read: %v %v", found, err)
|
||||
}
|
||||
if want, _ := BuildAskedAt(old.ID); !at.Equal(want) {
|
||||
t.Fatalf("read %v, asked %v", at, want)
|
||||
}
|
||||
for range 3 {
|
||||
if _, _, err := gate.OldestWaiting(ctx, q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(q.worker.Stream, q.worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Delivered.Consumer != 0 || info.NumAckPending != 0 || info.NumPending != 1 {
|
||||
t.Fatalf("reading the age took the ask: delivered %d, pending acks %d, waiting %d", info.Delivered.Consumer, info.NumAckPending, info.NumPending)
|
||||
}
|
||||
// And an empty queue reads as none.
|
||||
if _, found, err := gate.OldestWaiting(ctx, &takeQueue{kind: queueReady, worker: broker.CheckWorkers()[0]}); err != nil || found {
|
||||
t.Fatalf("an empty queue read as %v (%v)", found, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687 and 1688).
|
||||
//
|
||||
// **The layout other parts read** (ADR 0484 clause 12 has the checker read the same key; a layout shared between modules
|
||||
// belongs in mesh-sdk, and this one is not there yet):
|
||||
//
|
||||
// - bucket `mesh-controller_plans-building` (broker.PlansBuildingBucket): one key per delivery plan in its build phase, named
|
||||
// by the plan's id (`plan-<digits>`), value a PlanBuilding as JSON. Written by the controller (lease holder) alone: put when
|
||||
// the plan's first build is asked, put again about every thirty seconds, deleted when the phase ends; the bucket's age
|
||||
// (two minutes) removes the key of a controller that stopped. A reader lists the keys and needs no value to decide.
|
||||
// - key `let-through` in bucket `mesh-controller_checks-running` (broker.CheckRunningBucket), value a LetThrough as JSON:
|
||||
// created by the holder that takes a check which waited LetThroughAfter, refreshed at each heartbeat of that check, deleted
|
||||
// when it ends. At most one such check runs at a time on the build seat, because the create fails while the key is there.
|
||||
|
||||
// LetThroughAfter is how long a check waits, while a delivery builds, before it is let through (ADR 0483 clause 3).
|
||||
const LetThroughAfter = 30 * time.Minute
|
||||
|
||||
// LetThroughKey is the key of the one check let through, in the bucket of the checks running. It has no dot, so it is no
|
||||
// pull request's key (`<owner>.<repo>.<number>`) and no group's (`group.<name>`).
|
||||
const LetThroughKey = "let-through"
|
||||
|
||||
// PlanBuilding is the value of a plan's key in the bucket of the plans building.
|
||||
type PlanBuilding struct {
|
||||
Plan string `json:"plan"`
|
||||
Since time.Time `json:"since"`
|
||||
Seen time.Time `json:"seen"`
|
||||
}
|
||||
|
||||
// LetThrough is the value of the let-through key: the builder that runs the check and when it took it.
|
||||
type LetThrough struct {
|
||||
Builder string `json:"builder"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// checkGate is what a taking loop asks of the bus to hold checks while a delivery builds; a variable of the machine so a test
|
||||
// hands out against fakes.
|
||||
type checkGate interface {
|
||||
// Building is the ids of the plans in their build phase: the keys of the bucket. An error is a bucket that cannot be read.
|
||||
Building(ctx context.Context) ([]string, error)
|
||||
// OldestWaiting is when the first waiting ask of a check queue was asked, read without taking it.
|
||||
OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error)
|
||||
// TakeLetThrough creates the let-through key for a builder: false when it is there (another builder runs a check let through).
|
||||
TakeLetThrough(ctx context.Context, builder string) (bool, error)
|
||||
// KeepLetThrough puts the key again, and DropLetThrough deletes it.
|
||||
KeepLetThrough(ctx context.Context, builder string) error
|
||||
DropLetThrough(ctx context.Context) error
|
||||
}
|
||||
|
||||
// busGate is the gate over a connection.
|
||||
type busGate struct{ conn *nats.Conn }
|
||||
|
||||
func (g busGate) Building(ctx context.Context) ([]string, error) {
|
||||
return BuildingPlans(ctx, g.conn)
|
||||
}
|
||||
|
||||
// BuildingPlans is the ids of the delivery plans in their build phase, from the controller's bucket. A reader that is not the
|
||||
// build seat's (the checker, ADR 0484 clause 12) calls this. A bucket that cannot be read is an error; none is none building.
|
||||
func BuildingPlans(ctx context.Context, conn *nats.Conn) ([]string, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// OldestWaiting reads the first ask after the last one the check queue's worker handed out: a direct get on the stream, which
|
||||
// takes nothing from it and counts against no delivery. The ask's own id says when it was asked.
|
||||
func (g busGate) OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
info, err := jsOf(g.conn).ConsumerInfo(q.worker.Stream, q.worker.Name, nats.Context(ctx))
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the %s worker: %w", q.kind, err)
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"seq": info.Delivered.Stream + 1, "next_by_subj": q.worker.Filters[0]})
|
||||
reply, err := g.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+q.worker.Stream, body)
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the first waiting ask of the %s queue: %w", q.kind, err)
|
||||
}
|
||||
if reply.Header.Get("Status") == "404" || reply.Header.Get("Status") == "408" {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return askedAtOf(reply.Data, reply.Header.Get("Nats-Time-Stamp")), true, nil
|
||||
}
|
||||
|
||||
// askedAtOf is when a request was asked: its id says (NewBuildID), the stream's own time otherwise, and a request that says
|
||||
// neither is taken as asked now, so it is never let through on a guess.
|
||||
func askedAtOf(data []byte, stored string) time.Time {
|
||||
var r BuildRequest
|
||||
if json.Unmarshal(data, &r) == nil {
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
return at
|
||||
}
|
||||
}
|
||||
if at, err := time.Parse(time.RFC3339Nano, stored); err == nil {
|
||||
return at
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func jsOf(conn *nats.Conn) nats.JetStreamContext {
|
||||
js, _ := conn.JetStream()
|
||||
return js
|
||||
}
|
||||
|
||||
func (g busGate) TakeLetThrough(ctx context.Context, builder string) (bool, error) {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
if _, err := kv.Create(ctx, LetThroughKey, body); err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g busGate) KeepLetThrough(ctx context.Context, builder string) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
_, err = kv.Put(ctx, LetThroughKey, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (g busGate) DropLetThrough(ctx context.Context) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, LetThroughKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckHold is what a reader of the queue (the controller's own verb) says of why checks wait: the plans building and the check
|
||||
// let through now, if one.
|
||||
type CheckHold struct {
|
||||
Plans []string
|
||||
LetThrough *LetThrough
|
||||
}
|
||||
|
||||
// ReadCheckHold reads the plans building and the let-through key. A bucket that cannot be read is an error the caller says.
|
||||
func ReadCheckHold(ctx context.Context, conn *nats.Conn) (CheckHold, error) {
|
||||
var h CheckHold
|
||||
plans, err := BuildingPlans(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
h.Plans = plans
|
||||
kv, err := runningBucket(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
e, err := kv.Get(ctx, LetThroughKey)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return h, nil
|
||||
}
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
var l LetThrough
|
||||
if json.Unmarshal(e.Value(), &l) == nil {
|
||||
h.LetThrough = &l
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// gateOf is the machine's gate: the one a test set, or the bus's.
|
||||
func (m *natsMachine) gateOf() checkGate {
|
||||
if m.gate != nil {
|
||||
return m.gate
|
||||
}
|
||||
return busGate{conn: m.js.Conn()}
|
||||
}
|
||||
|
||||
// sayOnce says a fault of the hold the first time it happens and again once it has recovered and happened anew, so a bucket
|
||||
// that cannot be read is in the log once and not every second.
|
||||
func (m *natsMachine) sayOnce(kind string, err error) {
|
||||
if m.said == nil {
|
||||
m.said = map[string]bool{}
|
||||
}
|
||||
if err == nil {
|
||||
delete(m.said, kind)
|
||||
return
|
||||
}
|
||||
if !m.said[kind] {
|
||||
m.said[kind] = true
|
||||
fmt.Fprintf(os.Stderr, "%s; read as empty, so no check is held for it: %v\n", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// building says whether a delivery builds: any key in the bucket of the plans building. **A bucket that cannot be read is said
|
||||
// and read as empty** (ADR 0483 clause 2): a check wrongly held costs a pull request its turn, a fault that held every check
|
||||
// would stop the mesh checking.
|
||||
func (m *natsMachine) building(ctx context.Context) []string {
|
||||
plans, err := m.gateOf().Building(ctx)
|
||||
m.sayOnce("the bucket of the plans building cannot be read", err)
|
||||
if err != nil {
|
||||
// empty-on-error: ADR 0483 clause 2 — a bucket that cannot be read is said (sayOnce) and read as empty, because a check wrongly held costs a pull request its turn and a fault that held every check would stop the mesh checking.
|
||||
return nil
|
||||
}
|
||||
return plans
|
||||
}
|
||||
|
||||
// letOneThrough takes the oldest waiting check of the two check queues once it waited LetThroughAfter, by the one let-through
|
||||
// key: nil when none waited that long, when another holder runs the check let through, or when the check was taken by
|
||||
// another holder in the moment. The caller owns the key while it returns a message and releases it with dropLetThrough.
|
||||
func (m *natsMachine) letOneThrough(ctx context.Context) (*nats.Msg, *takeQueue) {
|
||||
gate := m.gateOf()
|
||||
var oldest *takeQueue
|
||||
var oldestAt time.Time
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
m.sayOnce("the first waiting check of the "+q.kind+" queue cannot be read", err)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || at.Before(oldestAt) {
|
||||
oldest, oldestAt = q, at
|
||||
}
|
||||
}
|
||||
if oldest == nil || time.Since(oldestAt) < m.letThroughAfter() {
|
||||
return nil, nil
|
||||
}
|
||||
created, err := gate.TakeLetThrough(ctx, m.on)
|
||||
m.sayOnce("the let-through key cannot be taken", err)
|
||||
if err != nil || !created {
|
||||
return nil, nil
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
defer end()
|
||||
fetched, err := oldest.pull.Fetch(1, nats.Context(trying))
|
||||
if len(fetched) == 0 {
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.Canceled) {
|
||||
m.sayOnce("the "+oldest.kind+" check queue cannot be taken from", err)
|
||||
}
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
// Another holder took the one that was oldest in the moment between the read and the fetch: what came is a younger check,
|
||||
// handed back and the key released, never run during a delivery.
|
||||
if askedAtOf(fetched[0].Data, "").Add(m.letThroughAfter()).After(time.Now()) {
|
||||
_ = fetched[0].Nak()
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
m.letKey = true
|
||||
return fetched[0], oldest
|
||||
}
|
||||
|
||||
func (m *natsMachine) letThroughAfter() time.Duration {
|
||||
if m.opts.LetThroughAfter > 0 {
|
||||
return m.opts.LetThroughAfter
|
||||
}
|
||||
return LetThroughAfter
|
||||
}
|
||||
|
||||
// dropLetThrough deletes the let-through key this holder created.
|
||||
func (m *natsMachine) dropLetThrough(ctx context.Context) {
|
||||
m.letKey = false
|
||||
if err := m.gateOf().DropLetThrough(context.WithoutCancel(ctx)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not take the let-through key away; it ages out: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepLetThrough refreshes the key at every heartbeat while the check it was created for runs, and returns the end of that: it
|
||||
// stops the refreshing and deletes the key.
|
||||
func (m *natsMachine) keepLetThrough(ctx context.Context, every time.Duration) (end func()) {
|
||||
done := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finished)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
putting, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
if err := m.gateOf().KeepLetThrough(putting, m.on); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not keep the let-through key: %v\n", err)
|
||||
}
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(done)
|
||||
<-finished
|
||||
m.dropLetThrough(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// whyHeld words the plans a check is held for, for the log.
|
||||
func whyHeld(plans []string) string { return strings.Join(plans, ", ") }
|
||||
@@ -0,0 +1,307 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
|
||||
// fake queues and a fake bus.
|
||||
|
||||
// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
|
||||
type fakeQueue struct {
|
||||
mu sync.Mutex
|
||||
waiting []*nats.Msg
|
||||
fetched int
|
||||
}
|
||||
|
||||
func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.fetched++
|
||||
if len(f.waiting) == 0 {
|
||||
return nil, nats.ErrTimeout
|
||||
}
|
||||
m := f.waiting[0]
|
||||
f.waiting = f.waiting[1:]
|
||||
return []*nats.Msg{m}, nil
|
||||
}
|
||||
|
||||
func (f *fakeQueue) fetches() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.fetched
|
||||
}
|
||||
|
||||
// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
|
||||
func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
|
||||
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
func buildAsk() *nats.Msg {
|
||||
body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
// fakeGate is the bus as the loop sees it for the hold.
|
||||
type fakeGate struct {
|
||||
mu sync.Mutex
|
||||
plans []string
|
||||
plansErr error
|
||||
letKey bool
|
||||
taken int
|
||||
kept int
|
||||
dropped int
|
||||
oldestFrom map[*takeQueue]*fakeQueue
|
||||
}
|
||||
|
||||
func (g *fakeGate) Building(context.Context) ([]string, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.plans, g.plansErr
|
||||
}
|
||||
|
||||
// OldestWaiting reads the first waiting ask of the fake queue without taking it.
|
||||
func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
f := g.oldestFrom[q]
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if len(f.waiting) == 0 {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return askedAtOf(f.waiting[0].Data, ""), true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.letKey {
|
||||
return false, nil
|
||||
}
|
||||
g.letKey = true
|
||||
g.taken++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) KeepLetThrough(context.Context, string) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.kept++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) DropLetThrough(context.Context) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.letKey = false
|
||||
g.dropped++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) setPlans(plans ...string) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.plans = plans
|
||||
}
|
||||
|
||||
// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
|
||||
func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
|
||||
builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
|
||||
subs := []*takeQueue{
|
||||
{kind: queueBuilds, pull: builds, bound: true},
|
||||
{kind: queueReady, pull: ready, bound: true},
|
||||
{kind: queueOther, pull: other, bound: true},
|
||||
}
|
||||
gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
|
||||
m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
|
||||
return m, builds, ready, other, gate
|
||||
}
|
||||
|
||||
func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
|
||||
t.Helper()
|
||||
msg, q, err := m.next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return msg, q
|
||||
}
|
||||
|
||||
// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
|
||||
// (between two tiers).
|
||||
func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
|
||||
// The delivery's queue is empty: between two tiers.
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
|
||||
}
|
||||
if ready.fetches() != 0 || other.fetches() != 0 {
|
||||
t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
|
||||
}
|
||||
if builds.fetches() == 0 {
|
||||
t.Fatal("the delivery's queue was not looked at")
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery build is still taken first, and the check waits behind it.
|
||||
func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueBuilds {
|
||||
t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("a check queue was fetched from")
|
||||
}
|
||||
}
|
||||
|
||||
// It is taken once the key is gone (deleted at the phase's end, or expired).
|
||||
func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("taken while the key is there")
|
||||
}
|
||||
gate.setPlans()
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatalf("the check was not taken once the key was gone: %v", q)
|
||||
}
|
||||
if m.letKey {
|
||||
t.Fatal("a check taken without a delivery building took the let-through key")
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
|
||||
func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.plansErr = errors.New("permissions violation")
|
||||
gate.plans = []string{"plan-1"}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatal("a check was held because the bucket could not be read")
|
||||
}
|
||||
if !m.said["the bucket of the plans building cannot be read"] {
|
||||
t.Fatalf("the fault was not said: %v", m.said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
|
||||
// waiting delivery build; a younger one is never taken before it.
|
||||
func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
|
||||
t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
|
||||
}
|
||||
if !m.letKey || gate.taken != 1 {
|
||||
t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
|
||||
func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
|
||||
var order []string
|
||||
for range 3 {
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil {
|
||||
t.Fatalf("no check let through after %v", order)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msg.Data, &r)
|
||||
order = append(order, fmt.Sprint(r.Check.Number, q.kind))
|
||||
// While it runs another holder (the same fake gate, key held) takes builds only.
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
|
||||
if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
|
||||
t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
|
||||
}
|
||||
m.dropLetThrough(context.Background())
|
||||
}
|
||||
if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check younger than the bound is never let through, and no key is created for it.
|
||||
func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("a check of 29 minutes was let through")
|
||||
}
|
||||
if gate.taken != 0 {
|
||||
t.Fatal("a let-through key was created for a young check")
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("the age was read by a fetch; it must be read without taking")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
|
||||
// moment between the read and the fetch is handed back with the key released.
|
||||
func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
// The read sees the old check; before the fetch another holder takes it and a young one is first.
|
||||
gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
|
||||
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a young check was run during a delivery: %s", msg.Data)
|
||||
}
|
||||
if gate.letKey || m.letKey {
|
||||
t.Fatal("the let-through key was left held")
|
||||
}
|
||||
}
|
||||
|
||||
// The key of a check let through is refreshed while it runs and deleted when it ends.
|
||||
func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg == nil {
|
||||
t.Fatal("not let through")
|
||||
}
|
||||
end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
gate.mu.Lock()
|
||||
held, kept := gate.letKey, gate.kept
|
||||
gate.mu.Unlock()
|
||||
if !held || kept == 0 {
|
||||
t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
|
||||
}
|
||||
end()
|
||||
gate.mu.Lock()
|
||||
defer gate.mu.Unlock()
|
||||
if gate.letKey || m.letKey {
|
||||
t.Error("the key was not deleted when the check ended")
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,40 @@ type Queue struct {
|
||||
AckPending int `json:"ack-pending"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []QueuedAsk `json:"asks"`
|
||||
// Hold is why the checks wait: the delivery plans building and the check let through now (ADR 0483, issues 1687 and 1688);
|
||||
// HoldFault says the bucket could not be read, in which case Hold is empty.
|
||||
Hold CheckHold `json:"hold"`
|
||||
HoldFault string `json:"hold-fault,omitempty"`
|
||||
}
|
||||
|
||||
// LetThroughOf says which ask is let through under ADR 0483 clause 3: the check that waited LetThroughAfter or longer and is next
|
||||
// (the oldest waiting check across the two check queues, while a delivery builds), and the check in flight that was let through
|
||||
// (the let-through key names the machine that runs it). Empty when none.
|
||||
func (q Queue) LetThroughOf(now time.Time) (next, running string) {
|
||||
if len(q.Hold.Plans) == 0 && q.Hold.LetThrough == nil {
|
||||
return "", ""
|
||||
}
|
||||
var oldest *QueuedAsk
|
||||
for i := range q.Asks {
|
||||
a := &q.Asks[i]
|
||||
if a.State != AskWaiting || a.Request.Check == nil || a.AskedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || a.AskedAt.Before(oldest.AskedAt) {
|
||||
oldest = a
|
||||
}
|
||||
}
|
||||
if oldest != nil && len(q.Hold.Plans) > 0 && now.Sub(oldest.AskedAt) >= LetThroughAfter {
|
||||
next = oldest.ID
|
||||
}
|
||||
if q.Hold.LetThrough != nil {
|
||||
for _, a := range q.Asks {
|
||||
if a.State == AskInFlight && a.Request.Check != nil && a.On == q.Hold.LetThrough.Builder {
|
||||
running = a.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
return next, running
|
||||
}
|
||||
|
||||
// Of is the asks in one state, in queue order.
|
||||
@@ -319,6 +353,16 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
for _, r := range reads {
|
||||
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
|
||||
}
|
||||
// And why the checks wait, for the build seat that has them (ADR 0483 clause 5). A bucket that cannot be read is said by the
|
||||
// queue, which then names no plan.
|
||||
if seat == broker.TheBuildSeat {
|
||||
hold, err := ReadCheckHold(ctx, js.Conn())
|
||||
if err != nil {
|
||||
q.HoldFault = err.Error()
|
||||
} else {
|
||||
q.Hold = hold
|
||||
}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user