jschoubben
99285cb080
mesh/delivery merged: source.main moved to novox/mesh-host@1195cf45971e
mesh/check pass 99285cb080db on 1195cf45971e | unit 303 run, 1080 kept, 5 skipped | integration 306 run, 1082 kept, 4 skipped | over bound: unit (slow…
The bus node's engine writes the bus user list (ADR 0547 bridge); each write makes the bus reload its accounts, and for about three seconds every JetStream request meets no responders. An apply that failed then was kept failed until a key changed (mesh-vault on novox, 2026-10-25 23:04:24), holding every module that needs the secret seat. A transient bus failure is now retried with a backoff (2 s, 5 s, 15 s, 60 s, then every 5 min), the have saying retrying meanwhile; after the engine writes the user list, the transiently failed modules are tried again once the reload is over. A have write that met it is written again the same way. shorten keeps the head and the tail, the cause.
69 lines
2.5 KiB
Go
69 lines
2.5 KiB
Go
package link
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
)
|
|
|
|
// A transient failure of the bus: it did not answer for a moment, and the same request a little later is answered. The
|
|
// bus answers so while it reloads its accounts after the bus node's engine wrote its user list (novox/hq ADR 0547 §8,
|
|
// about three seconds: every JetStream request then meets "no responders"), and while a link reconnects. A part that
|
|
// meets one is retried, never left failed until a key changes (2026-10-25: mesh-vault on the bus node stayed failed at
|
|
// 23:04:24, and with it every module that needs the secret seat).
|
|
|
|
// transientErrs are the bus client's errors of a bus that does not answer for a moment.
|
|
var transientErrs = []error{
|
|
nats.ErrNoResponders, nats.ErrTimeout, nats.ErrConnectionClosed, nats.ErrConnectionReconnecting,
|
|
nats.ErrDisconnected, nats.ErrStaleConnection, nats.ErrJetStreamNotEnabled, nats.ErrJetStreamNotEnabledForAccount,
|
|
nats.ErrNoStreamResponse,
|
|
}
|
|
|
|
// transientWords are the same as words, for a failure that reaches its reader only as text (an apply's refusal).
|
|
var transientWords = []string{
|
|
"no responders available for request", "nats: timeout", "nats: connection closed", "nats: connection reconnecting",
|
|
"nats: server is disconnected", "nats: stale connection", "jetstream not enabled", "no response from stream",
|
|
"jetstream system temporarily unavailable",
|
|
}
|
|
|
|
// Transient says err is the bus not answering for a moment, by the client's error or its words.
|
|
func Transient(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
for _, e := range transientErrs {
|
|
if errors.Is(err, e) {
|
|
return true
|
|
}
|
|
}
|
|
return TransientWords(err.Error())
|
|
}
|
|
|
|
// TransientWords says a failure's words are those of the bus not answering for a moment. A refused permission, a digest
|
|
// that does not match or anything the module's own apply says is not.
|
|
func TransientWords(s string) bool {
|
|
s = strings.ToLower(s)
|
|
for _, w := range transientWords {
|
|
if strings.Contains(s, w) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// backoffs are the waits before the first retries of a transient failure; every later one waits the last.
|
|
var backoffs = []time.Duration{2 * time.Second, 5 * time.Second, 15 * time.Second, time.Minute, 5 * time.Minute}
|
|
|
|
// Backoff is the wait before the try-th retry (1, 2, …) of a transient failure: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
|
func Backoff(try int) time.Duration {
|
|
if try < 1 {
|
|
try = 1
|
|
}
|
|
if try > len(backoffs) {
|
|
return backoffs[len(backoffs)-1]
|
|
}
|
|
return backoffs[try-1]
|
|
}
|