Wait out a bus that answers nothing for a few seconds instead of losing the write (issue 774)

At each mesh-grants restart and user-list reload (16:37:24, 16:55:15) nats-server 2.11's reloadAuthorization left
the mesh's account without its imports, $JS.API.> among them, for a few seconds: requests had no responders,
JetStream calls said it was not enabled, and a build-test result on ace was lost.

Quick fix, chosen as asked; the thorough fix is a reload that keeps the account's imports, in the bus's holder.
The package busretry asks a request again, backing off within about 30 s, only while its error says it reached no
one (no responders, JetStream not enabled, no response from the stream, or either in words): nothing was done, so
even a create or an update is safe to ask again. A timeout or any answer is never asked again. After the bound it
is said on standard error and returned. Used by process.State's requests, process.Conn's calls (asked again only
while $JS.API.INFO says the bus itself answers nothing, so a callee that is not running still fails at once) and
asks, and the bundle's state requests through the runtime.

go/busretry/busretry.go

@@ -0,0 +1,69 @@
+// Package busretry waits out a bus that answers nothing for a few seconds (issue 774, the sweep of 2026-10-30: bus
+// blackouts at every user-list reload). nats-server 2.11's reloadAuthorization rebuilds the accounts, and for a few
+// seconds the mesh's account has none of its imports, $JS.API.> among them: every request is answered "no responders"
+// and every JetStream call "jetstream not enabled". A write or a call made in that window failed, and was lost.
+//
+// Only an error that says the request reached no one is waited out (Undelivered): nothing was done, so asking again
+// is safe even for a write that is not idempotent. A timeout, or any answer, is never asked again: the request may
+// have been delivered. The wait is bounded (Bound, about 30 s, backing off), and said on standard error when it ends
+// without an answer.
+//
+// This is the quick fix. The thorough one is a reload that keeps the account's imports, in the bus's holder.
+package busretry
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bound is how long a request that reached no one is asked again; FirstWait the first pause, doubled to MostWait.
+var (
+	Bound     = 30 * time.Second
+	FirstWait = 250 * time.Millisecond
+	MostWait  = 4 * time.Second
+)
+
+// Undelivered says err means the request reached no one: no responders, JetStream not enabled (for the account), or
+// no response from the stream after the client's own retries on no responders; also said in words by a runtime that
+// passed the error on.
+func Undelivered(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, jetstream.ErrJetStreamNotEnabled) ||
+		errors.Is(err, jetstream.ErrJetStreamNotEnabledForAccount) || errors.Is(err, jetstream.ErrNoStreamResponse) ||
+		errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) {
+		return true
+	}
+	words := strings.ToLower(err.Error())
+	return strings.Contains(words, "no responders") || strings.Contains(words, "jetstream not enabled")
+}
+
+// Do runs try, and again while its error says the request reached no one, until Bound has passed. blackout, when
+// given, is asked after each such error and Do stops waiting when it says the bus is not blacked out: a call nobody
+// answers because its callee is not running fails at once, as before. what names the request in the error.
+func Do[T any](what string, try func() (T, error), blackout func() bool) (T, error) {
+	v, err := try()
+	if !Undelivered(err) || (blackout != nil && !blackout()) {
+		return v, err
+	}
+	start, wait := time.Now(), FirstWait
+	for time.Since(start) < Bound {
+		time.Sleep(wait)
+		wait = min(wait*2, MostWait)
+		v, err = try()
+		if !Undelivered(err) || (blackout != nil && !blackout()) {
+			return v, err
+		}
+	}
+	err = fmt.Errorf("%s: the bus answered nothing for %s, asked again until then (a user-list reload drops the "+
+		"account's imports for a few seconds; this was longer): %w", what, time.Since(start).Round(time.Second), err)
+	fmt.Fprintf(os.Stderr, "[mesh-sdk] %v\n", err)
+	return v, err
+}

go/busretry/busretry_test.go

@@ -0,0 +1,48 @@
+package busretry
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Issue 774 (bus blackouts at every user-list reload): only a request that reached no one is asked again, within the
+// bound; a blackout probe that says the bus answers ends the wait at once.
+func TestABusThatAnswersNothingIsWaitedOut(t *testing.T) {
+	Bound, FirstWait, MostWait = 60*time.Millisecond, time.Millisecond, 4*time.Millisecond
+	for _, row := range []struct {
+		name      string
+		errs      []error // what each try answers; the last repeats
+		blackout  func() bool
+		tries     int // at least
+		atMost    int
+		delivered bool
+	}{
+		{"no responders, then an answer", []error{nats.ErrNoResponders, nats.ErrNoResponders, nil}, nil, 3, 3, true},
+		{"jetstream not enabled, then an answer", []error{jetstream.ErrJetStreamNotEnabled, nil}, nil, 2, 2, true},
+		{"said in words by a runtime", []error{errors.New("nats: no responders available for request"), nil}, nil, 2, 2, true},
+		{"a timeout may have been delivered: never asked again", []error{context.DeadlineExceeded}, nil, 1, 1, false},
+		{"an answer that refuses is an answer", []error{errors.New("refused")}, nil, 1, 1, false},
+		{"nobody answers on a bus that answers: the callee is not running", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false},
+		{"blacked out past the bound: said", []error{jetstream.ErrJetStreamNotEnabledForAccount}, func() bool { return true }, 3, 1000, false},
+	} {
+		tries := 0
+		_, err := Do("the test", func() (int, error) {
+			e := row.errs[min(tries, len(row.errs)-1)]
+			tries++
+			return tries, e
+		}, row.blackout)
+		if tries < row.tries || tries > row.atMost || (err == nil) != row.delivered {
+			t.Errorf("%s: %d tries, %v", row.name, tries, err)
+		}
+		if strings.HasPrefix(row.name, "blacked out") && (!Undelivered(err) || !strings.Contains(fmt.Sprint(err), "the bus answered nothing")) {
+			t.Errorf("%s: %v", row.name, err)
+		}
+	}
+}