unit 302 and integration 34, as at the merge base 45e143f; the buscalls test moves beside busstate's. Builds: design 47 G §419, §626 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
118 lines
5.8 KiB
Go
118 lines
5.8 KiB
Go
package busretry
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// Issue 774 (bus blackouts at every user-list reload): only a request that reached no one is asked again, within the
|
|
// bound; a blackout probe that says the bus answers ends the wait at once.
|
|
func subABusThatAnswersNothingIsWaitedOut(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name string
|
|
errs []error // what each try answers; the last repeats
|
|
blackout func() bool
|
|
tries int // at least
|
|
atMost int
|
|
delivered bool
|
|
repeatable bool // DoRepeatable, a call marked safe to make again (ADR 0668)
|
|
}{
|
|
{"no responders, then an answer", []error{nats.ErrNoResponders, nats.ErrNoResponders, nil}, nil, 3, 3, true, false},
|
|
{"jetstream not enabled, then an answer", []error{jetstream.ErrJetStreamNotEnabled, nil}, nil, 2, 2, true, false},
|
|
{"said in words by a runtime", []error{errors.New("nats: no responders available for request"), nil}, nil, 2, 2, true, false},
|
|
{"a timeout may have been delivered: never asked again", []error{context.DeadlineExceeded}, nil, 1, 1, false, false},
|
|
{"an answer that refuses is an answer", []error{errors.New("refused")}, nil, 1, 1, false, false},
|
|
{"nobody answers on a bus that answers: the callee is not running", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false, false},
|
|
{"blacked out past the bound: said", []error{jetstream.ErrJetStreamNotEnabledForAccount}, func() bool { return true }, 2, 1000, false, false},
|
|
// Issue 1966, ADR 0668: a call kept and safe to make again is asked again on a timeout, and on no responders
|
|
// from a callee that restarts on a bus that answers; Do still asks neither.
|
|
{"repeatable: timed out twice, then an answer", []error{context.DeadlineExceeded, nats.ErrTimeout, nil}, nil, 3, 3, true, true},
|
|
{"repeatable: no responders with no probe, then an answer", []error{nats.ErrNoResponders, nil}, nil, 2, 2, true, true},
|
|
{"repeatable: a callee's answer is an answer", []error{&Answered{Words: "context deadline exceeded"}}, nil, 1, 1, false, true},
|
|
{"repeatable: nobody answers and the probe says the bus answers", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false, true},
|
|
{"repeatable: no response from the stream, either client's, then an answer", []error{nats.ErrNoStreamResponse, jetstream.ErrNoStreamResponse, nil}, nil, 3, 3, true, true},
|
|
{"timed out past the bound: said", []error{context.DeadlineExceeded}, nil, 2, 1000, false, true},
|
|
} {
|
|
// A row that ends in an answer waits on a bound of seconds, which no slow machine reaches; only the row that
|
|
// runs past it has a short one, and asks only that it was asked again at least once.
|
|
bound := 10 * time.Second
|
|
if strings.Contains(row.name, "past the bound") {
|
|
bound = 100 * time.Millisecond
|
|
}
|
|
t.Cleanup(Waits(bound, time.Millisecond, 4*time.Millisecond))
|
|
tries := 0
|
|
do := Do[int]
|
|
if row.repeatable {
|
|
do = DoRepeatable[int]
|
|
}
|
|
_, err := do("the test", func() (int, error) {
|
|
e := row.errs[min(tries, len(row.errs)-1)]
|
|
tries++
|
|
return tries, e
|
|
}, row.blackout)
|
|
if tries < row.tries || tries > row.atMost || (err == nil) != row.delivered {
|
|
t.Errorf("%s: %d tries, %v", row.name, tries, err)
|
|
}
|
|
if strings.HasPrefix(row.name, "blacked out") && (!Undelivered(err) || !strings.Contains(fmt.Sprint(err), "the bus answered nothing")) {
|
|
t.Errorf("%s: %v", row.name, err)
|
|
}
|
|
if strings.HasPrefix(row.name, "timed out") && (!TimedOut(err) || !strings.Contains(fmt.Sprint(err), "the test: no answer within its deadline")) {
|
|
t.Errorf("%s: %v", row.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Design 47 G §351, §460: a kept call is asked again only while its key is open and not replaced; an error of `open` is
|
|
// logged naming the key and the ask is made again.
|
|
func subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen(t *testing.T) {
|
|
for _, row := range []struct {
|
|
name string
|
|
open func(n int) (bool, error) // by the number of asks made so far
|
|
replaced bool
|
|
tries int
|
|
logged bool
|
|
}{
|
|
{"open throughout, then answered", func(int) (bool, error) { return true, nil }, false, 3, false},
|
|
{"closed after the first ask: not asked again", func(int) (bool, error) { return false, nil }, false, 1, false},
|
|
{"replaced: not asked again", func(int) (bool, error) { return true, nil }, true, 1, false},
|
|
{"open errs: logged and asked again", func(int) (bool, error) { return false, errors.New("state unreadable") }, false, 3, true},
|
|
} {
|
|
t.Cleanup(Waits(10*time.Second, time.Millisecond, 4*time.Millisecond))
|
|
tries, logs := 0, []string{}
|
|
o := Open{Key: "ask-7", Open: func(k string) (bool, error) {
|
|
if k != "ask-7" {
|
|
t.Errorf("%s: open asked about %q", row.name, k)
|
|
}
|
|
return row.open(tries)
|
|
}, Replaced: func() bool { return row.replaced },
|
|
Logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
|
_, err := DoKept("the test", o, func() (int, error) {
|
|
tries++
|
|
if tries < 3 {
|
|
return 0, nats.ErrNoResponders
|
|
}
|
|
return tries, nil
|
|
}, nil)
|
|
if tries != row.tries || (row.tries == 3) != (err == nil) || (len(logs) > 0) != row.logged ||
|
|
(row.logged && !strings.Contains(logs[0], "ask-7")) {
|
|
t.Errorf("%s: %d asks, err %v, logs %v", row.name, tries, err, logs)
|
|
}
|
|
}
|
|
if _, err := DoKept("x", Open{Key: "k"}, func() (int, error) { return 0, nil }, nil); err == nil {
|
|
t.Errorf("a kept call with no open function was made")
|
|
}
|
|
}
|
|
|
|
// TestABusThatAnswersNothingIsWaitedOut runs the tests of this package that were merged to keep the module within its test budget (design 47 G §419), each as a subtest.
|
|
func TestABusThatAnswersNothingIsWaitedOut(t *testing.T) {
|
|
t.Run("ABusThatAnswersNothingIsWaitedOut", subABusThatAnswersNothingIsWaitedOut)
|
|
t.Run("AKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen", subAKeptCallIsAskedAgainOnlyWhileItsKeyIsOpen)
|
|
}
|