Files
mesh-sdk/go/busretry/busretry_test.go
T
jschoubben e9481928f5 KeptPace from settings: a module's declared pace paces its kept calls
busretry.ReadPace sets the kept-call pace from kept-call-pace-first and
kept-call-pace-max through the one settings reader; process.Connect and a
bundle's Serve read it at their start and refuse a value that is no
duration.

Builds: design 47 H2 §775, §781; H-A-370
2026-11-21 23:59:04 +01:00

240 lines
11 KiB
Go

package busretry
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// Issue 774 (bus blackouts at every user-list reload): only a request that reached no one is asked again, within the
// bound; a blackout probe that says the bus answers ends the wait at once.
func subABusThatAnswersNothingIsWaitedOut(t *testing.T) {
for _, row := range []struct {
name string
errs []error // what each try answers; the last repeats
blackout func() bool
tries int // at least
atMost int
delivered bool
repeatable bool // DoRepeatable, a call marked safe to make again (ADR 0668)
}{
{"no responders, then an answer", []error{nats.ErrNoResponders, nats.ErrNoResponders, nil}, nil, 3, 3, true, false},
{"jetstream not enabled, then an answer", []error{jetstream.ErrJetStreamNotEnabled, nil}, nil, 2, 2, true, false},
{"said in words by a runtime", []error{errors.New("nats: no responders available for request"), nil}, nil, 2, 2, true, false},
{"a timeout may have been delivered: never asked again", []error{context.DeadlineExceeded}, nil, 1, 1, false, false},
{"an answer that refuses is an answer", []error{errors.New("refused")}, nil, 1, 1, false, false},
{"nobody answers on a bus that answers: the callee is not running", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false, false},
{"blacked out past the bound: said", []error{jetstream.ErrJetStreamNotEnabledForAccount}, func() bool { return true }, 2, 1000, false, false},
// Issue 1966, ADR 0668: a call kept and safe to make again is asked again on a timeout, and on no responders
// from a callee that restarts on a bus that answers; Do still asks neither.
{"repeatable: timed out twice, then an answer", []error{context.DeadlineExceeded, nats.ErrTimeout, nil}, nil, 3, 3, true, true},
{"repeatable: no responders with no probe, then an answer", []error{nats.ErrNoResponders, nil}, nil, 2, 2, true, true},
{"repeatable: a callee's answer is an answer", []error{&Answered{Words: "context deadline exceeded"}}, nil, 1, 1, false, true},
{"repeatable: nobody answers and the probe says the bus answers", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false, true},
{"repeatable: no response from the stream, either client's, then an answer", []error{nats.ErrNoStreamResponse, jetstream.ErrNoStreamResponse, nil}, nil, 3, 3, true, true},
{"timed out past the bound: said", []error{context.DeadlineExceeded}, nil, 2, 1000, false, true},
} {
// A row that ends in an answer waits on a bound of seconds, which no slow machine reaches; only the row that
// runs past it has a short one, and asks only that it was asked again at least once.
bound := 10 * time.Second
if strings.Contains(row.name, "past the bound") {
bound = 100 * time.Millisecond
}
t.Cleanup(Waits(bound, time.Millisecond, 4*time.Millisecond))
tries := 0
do := Do[int]
if row.repeatable {
do = DoRepeatable[int]
}
_, err := do("the test", func() (int, error) {
e := row.errs[min(tries, len(row.errs)-1)]
tries++
return tries, e
}, row.blackout)
if tries < row.tries || tries > row.atMost || (err == nil) != row.delivered {
t.Errorf("%s: %d tries, %v", row.name, tries, err)
}
if strings.HasPrefix(row.name, "blacked out") && (!Undelivered(err) || !strings.Contains(fmt.Sprint(err), "the bus answered nothing")) {
t.Errorf("%s: %v", row.name, err)
}
if strings.HasPrefix(row.name, "timed out") && (!TimedOut(err) || !strings.Contains(fmt.Sprint(err), "the test: no answer within its deadline")) {
t.Errorf("%s: %v", row.name, err)
}
}
}
// Design 47 G §351, §460: a kept call is asked again only while its key is open and not replaced; an error of `open` is
// logged naming the key and the ask is made again.
func subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen(t *testing.T) {
for _, row := range []struct {
name string
open func(n int) (bool, error) // by the number of asks made so far
replaced bool
tries int
logged bool
}{
{"open throughout, then answered", func(int) (bool, error) { return true, nil }, false, 3, false},
{"closed after the first ask: not asked again", func(int) (bool, error) { return false, nil }, false, 1, false},
{"replaced: not asked again", func(int) (bool, error) { return true, nil }, true, 1, false},
{"open errs: logged and asked again", func(int) (bool, error) { return false, errors.New("state unreadable") }, false, 3, true},
} {
t.Cleanup(Waits(10*time.Second, time.Millisecond, 4*time.Millisecond))
tries, logs := 0, []string{}
o := Open{Key: "ask-7", Open: func(k string) (bool, error) {
if k != "ask-7" {
t.Errorf("%s: open asked about %q", row.name, k)
}
return row.open(tries)
}, Replaced: func() bool { return row.replaced },
Logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
_, err := DoKept("the test", o, func() (int, error) {
tries++
if tries < 3 {
return 0, nats.ErrNoResponders
}
return tries, nil
}, nil)
if tries != row.tries || (row.tries == 3) != (err == nil) || (len(logs) > 0) != row.logged ||
(row.logged && !strings.Contains(logs[0], "ask-7")) {
t.Errorf("%s: %d asks, err %v, logs %v", row.name, tries, err, logs)
}
}
if _, err := DoKept("x", Open{Key: "k"}, func() (int, error) { return 0, nil }, nil); err == nil {
t.Errorf("a kept call with no open function was made")
}
}
// fakeClock replaces the paced form's clock and sleep for one test: a sleep moves the clock, nothing sleeps.
func fakeClock(t *testing.T) (slept *[]time.Duration) {
at := time.Date(2026, 11, 21, 22, 0, 0, 0, time.UTC)
waits := []time.Duration{}
wasNow, wasSleep, wasJournal := now, sleep, journal
now = func() time.Time { return at }
sleep = func(d time.Duration) { waits = append(waits, d); at = at.Add(d) }
journal = &faults{seats: map[string]*seatFaults{}}
t.Cleanup(func() { now, sleep, journal = wasNow, wasSleep, wasJournal })
return &waits
}
// Design 47 H2 §775: the kept call is paced from 1 s doubling to 30 s, each wait within 20 %; met by no responders twice
// and then answered it returns the answer and writes one journal line; past its caller's bound it returns every try;
// a refusal returns at once; the journal says a seat at most once a minute.
func subAKeptCallIsPaced(t *testing.T) {
t.Run("no responders twice, then answered: the answer and one line", func(t *testing.T) {
slept := fakeClock(t)
tries, logs := 0, []string{}
v, err := DoPaced(Kept{Seat: "mesh-builds", Verb: "build", Ask: "a1", Bound: time.Minute,
Logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}, func() (int, error) {
if tries++; tries <= 2 {
return 0, nats.ErrNoResponders
}
return 7, nil
})
if v != 7 || err != nil || tries != 3 || len(logs) != 1 || !strings.Contains(logs[0], "mesh-builds") ||
!strings.Contains(logs[0], "build") || !strings.Contains(logs[0], "[a1]") {
t.Fatalf("%d, %v after %d tries, logs %q", v, err, tries, logs)
}
for i, base := range []time.Duration{time.Second, 2 * time.Second} {
if w := (*slept)[i]; w < base*8/10 || w > base*12/10 {
t.Errorf("wait %d is %s, not within 20 %% of %s", i, w, base)
}
}
})
t.Run("past its bound: every try", func(t *testing.T) {
slept := fakeClock(t)
tries := 0
_, err := DoPaced(Kept{Seat: "mesh-wants", Verb: "compose-candidate", Ask: "a2", Bound: 3 * time.Minute,
Logf: func(string, ...any) {}}, func() (int, error) { tries++; return 0, context.DeadlineExceeded })
var u *Unanswered
if !errors.As(err, &u) || len(u.Tries) != tries || tries < 5 || !TimedOut(err) ||
!strings.Contains(err.Error(), "context deadline exceeded") {
t.Fatalf("%d tries: %v", tries, err)
}
for _, w := range *slept {
if w > 30*time.Second*12/10 {
t.Errorf("a wait of %s passes the pace's most", w)
}
}
})
t.Run("a refusal returns at once", func(t *testing.T) {
slept := fakeClock(t)
tries := 0
_, err := DoPaced(Kept{Seat: "git", Verb: "merge"}, func() (int, error) { tries++; return 0, &Answered{Words: "refused"} })
if tries != 1 || len(*slept) != 0 || err == nil {
t.Fatalf("%d tries, %v", tries, err)
}
})
t.Run("away is a transport fault; its answer reads back", func(t *testing.T) {
raw, _ := json.Marshal(AwayAnswer("the forge's store", "401 and the version read failed"))
a := AwayIn(raw)
if a == nil || a.Part != "the forge's store" || a.Words != "401 and the version read failed" || !Fault(a) ||
!strings.Contains(string(raw), `"error-kind":"away"`) {
t.Fatalf("%s read as %+v", raw, a)
}
if AwayIn([]byte(`{"error":"no","error-kind":"refused"}`)) != nil || AwayIn([]byte(`[1]`)) != nil || Fault(&Answered{Words: "away"}) {
t.Fatal("a refusal reads as away")
}
if a := AwayIn([]byte(`{"error":"the store is down","error-kind":"away"}`)); a == nil || a.Words != "the store is down" {
t.Fatalf("an away answer with no words reads %+v", a)
}
})
t.Run("the pace is the module's settings", func(t *testing.T) {
was := KeptPace
t.Cleanup(func() { KeptPace = was })
for _, c := range []struct {
set map[string]time.Duration
bad bool
want Pace
}{
{set: nil, want: was},
{set: map[string]time.Duration{PaceFirstSetting: 2 * time.Second, PaceMaxSetting: time.Minute}, want: Pace{First: 2 * time.Second, Max: time.Minute}},
{set: map[string]time.Duration{PaceFirstSetting: time.Minute, PaceMaxSetting: time.Second}, bad: true, want: was},
{set: map[string]time.Duration{PaceFirstSetting: -1}, bad: true, want: was},
} {
KeptPace = was
err := ReadPace(func(key string, def time.Duration) (time.Duration, error) {
if d, ok := c.set[key]; ok {
if d < 0 {
return 0, fmt.Errorf("the setting %s is no duration", key)
}
return d, nil
}
return def, nil
})
if (err != nil) != c.bad || KeptPace != c.want {
t.Errorf("settings %v: pace %+v, %v", c.set, KeptPace, err)
}
}
})
t.Run("one line per seat a minute", func(t *testing.T) {
fakeClock(t)
logs := 0
k := Kept{Seat: "mesh-builds", Verb: "build", Bound: 10 * time.Second, Logf: func(string, ...any) { logs++ }}
fail := func() (int, error) { return 0, nats.ErrNoResponders }
_, _ = DoPaced(k, fail)
_, _ = DoPaced(k, fail)
if logs != 1 {
t.Fatalf("%d lines within a minute", logs)
}
k.Bound = 2 * time.Minute
_, _ = DoPaced(k, fail)
if logs < 2 {
t.Fatalf("%d lines past a minute", logs)
}
})
}
// TestABusThatAnswersNothingIsWaitedOut runs the tests of this package that were merged to keep the module within its test budget (design 47 G §419), each as a subtest.
func TestABusThatAnswersNothingIsWaitedOut(t *testing.T) {
t.Run("ABusThatAnswersNothingIsWaitedOut", subABusThatAnswersNothingIsWaitedOut)
t.Run("AKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen", subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen)
t.Run("AKeptCallIsPaced", subAKeptCallIsPaced)
}