busretry.ReadPace sets the kept-call pace from kept-call-pace-first and kept-call-pace-max through the one settings reader; process.Connect and a bundle's Serve read it at their start and refuse a value that is no duration. Builds: design 47 H2 §775, §781; H-A-370
421 lines
16 KiB
Go
421 lines
16 KiB
Go
// Package busretry waits out a bus that answers nothing for a few seconds (issue 774, the sweep of 2026-10-30: bus
|
|
// blackouts at every user-list reload). nats-server 2.11's reloadAuthorization rebuilds the accounts, and for a few
|
|
// seconds the mesh's account has none of its imports, $JS.API.> among them: every request is answered "no responders"
|
|
// and every JetStream call "jetstream not enabled". A write or a call made in that window failed, and was lost.
|
|
//
|
|
// Only an error that says the request reached no one is waited out (Undelivered): nothing was done, so asking again
|
|
// is safe even for a write that is not idempotent. A timeout, or any answer, is never asked again: the request may
|
|
// have been delivered. The wait is bounded (Bound, about 30 s, backing off), and said on standard error when it ends
|
|
// without an answer.
|
|
//
|
|
// DoRepeatable is the form for a call its caller keeps the answer of and marks safe to make again (a read, or a write
|
|
// carrying its own message id): it also asks again on a timeout, and its blackout probe is optional (novox/hq ADR 0668,
|
|
// extending ADR 0615 §6 to every kept call; issue 1966).
|
|
package busretry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math/rand/v2"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// Bound is how long a request that reached no one is asked again; FirstWait the first pause, doubled to MostWait.
|
|
var (
|
|
Bound = 30 * time.Second
|
|
FirstWait = 250 * time.Millisecond
|
|
MostWait = 4 * time.Second
|
|
)
|
|
|
|
// Waits sets Bound, FirstWait and MostWait and returns what puts the earlier values back. They are state every caller
|
|
// in a process shares, so a test sets them only through it, as t.Cleanup(busretry.Waits(...)): a test that set them
|
|
// and left them set shortened the next test's bound to milliseconds, and a loaded check machine overran it (the check
|
|
// of release sdk1, 2026-11-02). A bound a test waits on to succeed is seconds: no slow machine reaches it.
|
|
// It sets the kept-call pace (KeptPace) to first and most too, so a test of a kept call waits milliseconds.
|
|
func Waits(bound, first, most time.Duration) (restore func()) {
|
|
was := [3]time.Duration{Bound, FirstWait, MostWait}
|
|
wasPace := KeptPace
|
|
Bound, FirstWait, MostWait = bound, first, most
|
|
KeptPace = Pace{First: first, Max: most}
|
|
return func() { Bound, FirstWait, MostWait, KeptPace = was[0], was[1], was[2], wasPace }
|
|
}
|
|
|
|
// Pace is the kept-call form's pace (design 47 H2 §775): on a transport fault the call is made again after First,
|
|
// the wait doubling to at most Max, each wait drawn within 20 % of its value so many callers do not call in step.
|
|
type Pace struct{ First, Max time.Duration }
|
|
|
|
// The settings every module using the kept-call form declares, with DefaultPace's values as their defaults (§775).
|
|
const (
|
|
PaceFirstSetting = "kept-call-pace-first"
|
|
PaceMaxSetting = "kept-call-pace-max"
|
|
)
|
|
|
|
// DefaultPace is the pace a module's settings name when they name none (§775): 1 s doubling to 30 s, so a holder
|
|
// between two lives (a restart takes 20 to 50 s) is met within one minute without a call a second.
|
|
var DefaultPace = Pace{First: time.Second, Max: 30 * time.Second}
|
|
|
|
// KeptPace is the pace of every kept call this process makes: DefaultPace until the process reads its settings
|
|
// (ReadPace, at process.Connect and at a bundle's stdio.Serve).
|
|
var KeptPace = DefaultPace
|
|
|
|
// ReadPace sets KeptPace from the module's settings PaceFirstSetting and PaceMaxSetting, read through read (the one
|
|
// settings reader, stdio.SettingDuration, which answers its default for a setting not composed), so a module's
|
|
// declared pace paces its kept calls (§775). A setting that is no duration, or a most below the first, is an error
|
|
// naming it, and KeptPace stays.
|
|
func ReadPace(read func(key string, def time.Duration) (time.Duration, error)) error {
|
|
first, err := read(PaceFirstSetting, KeptPace.First)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
most, err := read(PaceMaxSetting, KeptPace.Max)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if most < first {
|
|
return fmt.Errorf("the setting %s (%s) is below %s (%s)", PaceMaxSetting, most, PaceFirstSetting, first)
|
|
}
|
|
if read := (Pace{First: first, Max: most}); read != KeptPace {
|
|
KeptPace = read
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// The clock and the sleep the paced form waits on; a test of this package replaces both, so nothing sleeps.
|
|
var (
|
|
now = time.Now
|
|
sleep = time.Sleep
|
|
)
|
|
|
|
// jittered is w drawn within 20 % of its value.
|
|
func jittered(w time.Duration) time.Duration {
|
|
return time.Duration(float64(w) * (0.8 + 0.4*rand.Float64()))
|
|
}
|
|
|
|
// Kept names one kept call of the paced form (design 47 H2 §775): the seat and the verb it asks (named in the journal
|
|
// line), the ask's id (named among the asks waiting), the caller's bound (zero is Bound) and the pace (zero is
|
|
// KeptPace). Open, when given, makes it asked again only while its key is open (DoKept). Logf is the journal; nil
|
|
// writes to standard error.
|
|
type Kept struct {
|
|
Seat, Verb, Ask string
|
|
Bound time.Duration
|
|
Pace Pace
|
|
Open *Open
|
|
Logf func(format string, args ...any)
|
|
}
|
|
|
|
// Try is one call of a kept call that met a transport fault: when it was made and the words it met.
|
|
type Try struct {
|
|
At time.Time
|
|
Words string
|
|
}
|
|
|
|
// Unanswered is a kept call's end past its caller's bound (§775): every call's time and words, so the caller's
|
|
// could-not-run or could-not-be-performed names them. It wraps the last fault, so Undelivered and TimedOut read it.
|
|
type Unanswered struct {
|
|
Seat, Verb, Ask string
|
|
Bound time.Duration
|
|
Tries []Try
|
|
Last error
|
|
}
|
|
|
|
func (u *Unanswered) Error() string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "%s.%s", u.Seat, u.Verb)
|
|
if u.Ask != "" {
|
|
fmt.Fprintf(&b, " (ask %s)", u.Ask)
|
|
}
|
|
fmt.Fprintf(&b, " unanswered within its bound of %s, called %d times:", u.Bound, len(u.Tries))
|
|
for i, t := range u.Tries {
|
|
if i > 0 {
|
|
b.WriteString(";")
|
|
}
|
|
fmt.Fprintf(&b, " %s %s", t.At.UTC().Format("15:04:05.000"), t.Words)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func (u *Unanswered) Unwrap() error { return u.Last }
|
|
|
|
// Fault says err is a transport fault of a kept call (§775): a request that reached no one, a call's own deadline
|
|
// passing, or a holder's answer of the kind away (§776); a callee's refusal never is.
|
|
func Fault(err error) bool {
|
|
var away *Away
|
|
return TimedOut(err) || Undelivered(err) || errors.As(err, &away)
|
|
}
|
|
|
|
// ErrorKindAway is the seat error kind a holder answers when a part it depends on is away (design 47 H2 §776): its
|
|
// answer's `error-kind`, beside `part` (what did not answer) and `words`. A caller reads it as a transport fault.
|
|
const ErrorKindAway = "away"
|
|
|
|
// Away is a holder's answer of the kind away, read as an error: the part that did not answer and the holder's words.
|
|
type Away struct {
|
|
Part, Words string
|
|
}
|
|
|
|
func (a *Away) Error() string {
|
|
return fmt.Sprintf("away: %s does not answer: %s", a.Part, a.Words)
|
|
}
|
|
|
|
// AwayAnswer is the seat error answer of the kind away a holder gives (§776): `error` and `error-kind` as each seat's
|
|
// error answer has them, with `part` and `words`.
|
|
func AwayAnswer(part, words string) map[string]string {
|
|
return map[string]string{"error": (&Away{Part: part, Words: words}).Error(), "error-kind": ErrorKindAway,
|
|
"part": part, "words": words}
|
|
}
|
|
|
|
// AwayIn is the *Away a verb's answer says when its `error-kind` is away, else nil: an answer that is no object, or
|
|
// of another kind, is the caller's to read.
|
|
func AwayIn(answer json.RawMessage) *Away {
|
|
var a struct {
|
|
Error string `json:"error"`
|
|
Kind string `json:"error-kind"`
|
|
Part string `json:"part"`
|
|
Words string `json:"words"`
|
|
}
|
|
if json.Unmarshal(answer, &a) != nil || a.Kind != ErrorKindAway {
|
|
return nil
|
|
}
|
|
if a.Words == "" {
|
|
a.Words = a.Error
|
|
}
|
|
return &Away{Part: a.Part, Words: a.Words}
|
|
}
|
|
|
|
// DoPaced makes a kept call through try (design 47 H2 §775): on a transport fault (Fault) it calls again at k's pace,
|
|
// each wait jittered, until the caller's bound, writing one journal line per seat at the first fault and then at most
|
|
// once a minute (the seat, the verb, the asks waiting and the count of calls). Past the bound it returns the last
|
|
// answer with an *Unanswered naming every call; a refusal of what was asked returns at once.
|
|
func DoPaced[T any](k Kept, try func() (T, error)) (T, error) {
|
|
bound, pace := k.Bound, k.Pace
|
|
if bound <= 0 {
|
|
bound = Bound
|
|
}
|
|
if pace.First <= 0 {
|
|
pace.First = KeptPace.First
|
|
}
|
|
if pace.Max <= 0 {
|
|
pace.Max = KeptPace.Max
|
|
}
|
|
logf := k.Logf
|
|
if logf == nil {
|
|
logf = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-sdk] "+f+"\n", a...) }
|
|
}
|
|
start, wait := now(), pace.First
|
|
var tries []Try
|
|
waiting := false
|
|
defer func() {
|
|
if waiting {
|
|
journal.done(k.Seat, k.Ask)
|
|
}
|
|
}()
|
|
for {
|
|
at := now()
|
|
v, err := try()
|
|
if !Fault(err) || (k.Open != nil && !k.Open.stillOpen()) {
|
|
return v, err
|
|
}
|
|
tries = append(tries, Try{At: at, Words: err.Error()})
|
|
if !waiting {
|
|
journal.wait(k.Seat, k.Ask)
|
|
waiting = true
|
|
}
|
|
journal.fault(k.Seat, k.Verb, logf)
|
|
left := bound - now().Sub(start)
|
|
if left <= 0 {
|
|
return v, &Unanswered{Seat: k.Seat, Verb: k.Verb, Ask: k.Ask, Bound: bound, Tries: tries, Last: err}
|
|
}
|
|
sleep(min(jittered(wait), left))
|
|
wait = min(wait*2, pace.Max)
|
|
}
|
|
}
|
|
|
|
// journal holds, per seat, the asks of this process waiting on it through a fault, the calls made since its last
|
|
// line and when that line was written: one line per seat at the first fault, then at most once a minute (§775).
|
|
var journal = &faults{seats: map[string]*seatFaults{}}
|
|
|
|
type faults struct {
|
|
mu sync.Mutex
|
|
seats map[string]*seatFaults
|
|
}
|
|
|
|
type seatFaults struct {
|
|
asks map[string]int
|
|
calls int
|
|
last time.Time
|
|
}
|
|
|
|
func (f *faults) of(seat string) *seatFaults {
|
|
s := f.seats[seat]
|
|
if s == nil {
|
|
s = &seatFaults{asks: map[string]int{}}
|
|
f.seats[seat] = s
|
|
}
|
|
return s
|
|
}
|
|
|
|
func (f *faults) wait(seat, ask string) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.of(seat).asks[ask]++
|
|
}
|
|
|
|
func (f *faults) done(seat, ask string) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
s := f.of(seat)
|
|
if s.asks[ask]--; s.asks[ask] <= 0 {
|
|
delete(s.asks, ask)
|
|
}
|
|
}
|
|
|
|
func (f *faults) fault(seat, verb string, logf func(string, ...any)) {
|
|
f.mu.Lock()
|
|
s := f.of(seat)
|
|
s.calls++
|
|
t := now()
|
|
if !s.last.IsZero() && t.Sub(s.last) < time.Minute {
|
|
f.mu.Unlock()
|
|
return
|
|
}
|
|
s.last = t
|
|
var asks []string
|
|
n := 0
|
|
for a, c := range s.asks {
|
|
n += c
|
|
if a != "" {
|
|
asks = append(asks, a)
|
|
}
|
|
}
|
|
sort.Strings(asks)
|
|
calls := s.calls
|
|
s.calls = 0
|
|
f.mu.Unlock()
|
|
logf("kept call: the seat %s does not answer (%s): %d asks waiting %v, %d calls since the last line; called "+
|
|
"again at the kept-call pace until each caller's bound", seat, verb, n, asks, calls)
|
|
}
|
|
|
|
// Undelivered says err means the request reached no one: no responders, JetStream not enabled (for the account), or
|
|
// no response from the stream after the client's own retries on no responders (either JetStream client's: the old
|
|
// one's publish says it for no responders, review round 1 of issue 1966); also said in words by a runtime that
|
|
// passed the error on.
|
|
func Undelivered(err error) bool {
|
|
var answered *Answered
|
|
if err == nil || errors.As(err, &answered) {
|
|
return false
|
|
}
|
|
if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, jetstream.ErrJetStreamNotEnabled) ||
|
|
errors.Is(err, jetstream.ErrJetStreamNotEnabledForAccount) || errors.Is(err, jetstream.ErrNoStreamResponse) ||
|
|
errors.Is(err, nats.ErrNoStreamResponse) ||
|
|
errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) {
|
|
return true
|
|
}
|
|
words := strings.ToLower(err.Error())
|
|
return strings.Contains(words, "no responders") || strings.Contains(words, "jetstream not enabled")
|
|
}
|
|
|
|
// Answered is an error a callee answered with: the request was delivered, whatever its words say ("jetstream not
|
|
// enabled" in a callee's own answer), so Undelivered never counts it and Do never asks it again.
|
|
type Answered struct{ Words string }
|
|
|
|
func (a *Answered) Error() string { return a.Words }
|
|
|
|
// Do runs try, and again while its error says the request reached no one, until Bound has passed. blackout, when
|
|
// given, is asked after each such error and Do stops waiting when it says the bus is not blacked out: a call nobody
|
|
// answers because its callee is not running fails at once, as before. what names the request in the error.
|
|
func Do[T any](what string, try func() (T, error), blackout func() bool) (T, error) {
|
|
return do(what, try, func(err error) bool { return Undelivered(err) && (blackout == nil || blackout()) })
|
|
}
|
|
|
|
// DoRepeatable is Do for a call the caller marks safe to make again (a read, or a write carrying its message id, which
|
|
// the callee deduplicates): a timeout is asked again too, and with no blackout probe a request that reached no one is
|
|
// asked again whether or not the bus is blacked out, since a callee that is restarting answers nothing for seconds
|
|
// (novox/hq ADR 0668; issue 1966). Past Bound the error says which fault it last met.
|
|
func DoRepeatable[T any](what string, try func() (T, error), blackout func() bool) (T, error) {
|
|
return do(what, try, func(err error) bool {
|
|
return TimedOut(err) || (Undelivered(err) && (blackout == nil || blackout()))
|
|
})
|
|
}
|
|
|
|
// Open is what an asker passes a kept call beside the ask's key (design 47 G §351, §425, §460): Open reads the asker's own
|
|
// state and says whether the key is still open; Replaced, when the asker has one, says an event that replaces the ask was
|
|
// heard. The ask is made again only while Open answers yes and Replaced says no. An error of Open is written to Logf
|
|
// (standard error when nil), naming the key, and the ask is made again: a kept call is one the callee takes once by a
|
|
// key, so asking again is safe and not asking would be silent. Open is always given.
|
|
type Open struct {
|
|
Key string
|
|
Open func(key string) (bool, error)
|
|
Replaced func() bool
|
|
Logf func(format string, args ...any)
|
|
}
|
|
|
|
func (o Open) stillOpen() bool {
|
|
if o.Replaced != nil && o.Replaced() {
|
|
return false
|
|
}
|
|
open, err := o.Open(o.Key)
|
|
if err != nil {
|
|
logf := o.Logf
|
|
if logf == nil {
|
|
logf = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-sdk] "+f+"\n", a...) }
|
|
}
|
|
logf("busretry: whether the ask %q is still open could not be read, so it is asked again: %v", o.Key, err)
|
|
return true
|
|
}
|
|
return open
|
|
}
|
|
|
|
// DoKept is DoRepeatable for a kept call that is asked again only while its key is open and not replaced (Open).
|
|
func DoKept[T any](what string, o Open, try func() (T, error), blackout func() bool) (T, error) {
|
|
if o.Open == nil {
|
|
var zero T
|
|
return zero, fmt.Errorf("%s: a kept call names its key's open function", what)
|
|
}
|
|
return do(what, try, func(err error) bool {
|
|
return (TimedOut(err) || (Undelivered(err) && (blackout == nil || blackout()))) && o.stillOpen()
|
|
})
|
|
}
|
|
|
|
// TimedOut says err is a request's deadline passing without an answer: the request may have been delivered, so only a
|
|
// call marked safe to make again (DoRepeatable) asks it again. A callee's own answer (Answered) never counts.
|
|
func TimedOut(err error) bool {
|
|
var answered *Answered
|
|
if err == nil || errors.As(err, &answered) {
|
|
return false
|
|
}
|
|
return errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout)
|
|
}
|
|
|
|
func do[T any](what string, try func() (T, error), again func(error) bool) (T, error) {
|
|
v, err := try()
|
|
if !again(err) {
|
|
return v, err
|
|
}
|
|
start, wait := time.Now(), FirstWait
|
|
for time.Since(start) < Bound {
|
|
time.Sleep(wait)
|
|
wait = min(wait*2, MostWait)
|
|
v, err = try()
|
|
if !again(err) {
|
|
return v, err
|
|
}
|
|
}
|
|
if TimedOut(err) {
|
|
err = fmt.Errorf("%s: no answer within its deadline, asked again for %s: %w", what, time.Since(start).Round(time.Second), err)
|
|
} else {
|
|
err = fmt.Errorf("%s: the bus answered nothing for %s, asked again until then (a user-list reload drops the "+
|
|
"account's imports for a few seconds, a callee that restarts answers nothing; this was longer): %w", what,
|
|
time.Since(start).Round(time.Second), err)
|
|
}
|
|
fmt.Fprintf(os.Stderr, "[mesh-sdk] %v\n", err)
|
|
return v, err
|
|
}
|