Retry an apply, a fetch by digest and a have write that met the bus reloading, and keep the cause in the have

The bus node's engine writes the bus user list (ADR 0547 bridge); each write makes the bus reload its accounts, and
for about three seconds every JetStream request meets no responders. An apply that failed then was kept failed until a
key changed (mesh-vault on novox, 2026-10-25 23:04:24), holding every module that needs the secret seat. A transient
bus failure is now retried with a backoff (2 s, 5 s, 15 s, 60 s, then every 5 min), the have saying retrying meanwhile;
after the engine writes the user list, the transiently failed modules are tried again once the reload is over. A have
write that met it is written again the same way. shorten keeps the head and the tail, the cause.

internal/agent/agent_test.go

@@ -109,10 +109,13 @@ type bed struct {
 	removed  []string
 	backups  []string
 	failWith string
-	backup   func(module string) Backup
-	kept     map[string][]json.RawMessage
-	others   [][]delivery.Composed
-	said     []string
+	// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
+	transient bool
+	failTimes int
+	backup    func(module string) Backup
+	kept      map[string][]json.RawMessage
+	others    [][]delivery.Composed
+	said      []string
 	// settled is each module Settled was told, and settledRan whether what runs held it then.
 	settled    []string
 	settledRan []bool
@@ -132,8 +135,8 @@ func newBed(t *testing.T) *bed {
 				name = NodeModule
 			}
 			b.applied = append(b.applied, name)
-			if b.failWith != "" {
-				return Result{Failed: true, Detail: b.failWith}
+			if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
+				return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
 			}
 			return Result{}
 		},
@@ -375,34 +378,74 @@ func TestStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing
 // A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
 // of either key.
 func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
-	b := newBed(t)
-	b.failWith = "web.x: no space"
-	stop := b.run()
-	defer stop()
-	fp := fpOf("a")
-	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
-	wk, wraw := wantFor(t, "web", fp, dig, "")
-	b.comps.put(ck, craw)
-	b.wants.put(wk, wraw)
-	eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
-	eventually(t, "the have does not say failed", func() bool {
-		h := b.haves.last("web")
-		return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
+	// A real failure is not retried by the clock, however short the backoff: only by a change of either key.
+	t.Run("a real failure", func(t *testing.T) {
+		b := newBed(t)
+		b.failWith = "web.x: no space"
+		b.env.Backoff = func(int) time.Duration { return time.Millisecond }
+		stop := b.run()
+		defer stop()
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+		wk, wraw := wantFor(t, "web", fp, dig, "")
+		b.comps.put(ck, craw)
+		b.wants.put(wk, wraw)
+		eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		eventually(t, "the have does not say failed", func() bool {
+			h := b.haves.last("web")
+			return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "no space") && h.FP == fp
+		})
+		// An unrelated key changes: not a retry.
+		ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
+		b.comps.put(ck9, craw9)
+		time.Sleep(150 * time.Millisecond)
+		if n := len(b.appliedList()); n != 1 {
+			t.Fatalf("a failed apply was retried by an unrelated key or the clock (%d applies)", n)
+		}
+		// The want is written again: the retry.
+		b.mu.Lock()
+		b.failWith = ""
+		b.mu.Unlock()
+		b.wants.put(wk, wraw)
+		eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
+		eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
 	})
-	// An unrelated key changes: not a retry.
-	ck9, craw9, _ := composition(t, "web", fpOf("9"), false, file("web", "x", "9"))
-	b.comps.put(ck9, craw9)
-	time.Sleep(150 * time.Millisecond)
-	if n := len(b.appliedList()); n != 1 {
-		t.Fatalf("a failed apply was retried by an unrelated key (%d applies)", n)
+	// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
+	// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
+	// retrying meanwhile, never failed.
+	for name, backoff := range map[string]func(int) time.Duration{
+		"a transient failure, by the backoff":    func(int) time.Duration { return time.Millisecond },
+		"a transient failure, by RetryTransient": nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			b := newBed(t)
+			b.failWith, b.transient, b.failTimes = "fetching the object: nats: no responders available for request", true, 1
+			b.env.Backoff = backoff
+			stop := b.run()
+			defer stop()
+			fp := fpOf("a")
+			ck, craw, dig := composition(t, "vault", fp, false, file("vault", "x", "1"))
+			wk, wraw := wantFor(t, "vault", fp, dig, "")
+			b.comps.put(ck, craw)
+			b.wants.put(wk, wraw)
+			eventually(t, "not applied", func() bool { return len(b.appliedList()) >= 1 })
+			if backoff == nil {
+				eventually(t, "the have does not say retrying", func() bool {
+					return strings.HasPrefix(b.haves.last("vault").Detail, "retrying")
+				})
+				b.ag.RetryTransient()
+			}
+			eventually(t, "not applied again without a new revision", func() bool { return len(b.appliedList()) == 2 })
+			eventually(t, "not running", func() bool { return b.haves.last("vault").Phase == string(delivery.PhaseRunning) })
+			b.haves.mu.Lock()
+			defer b.haves.mu.Unlock()
+			for _, h := range b.haves.log {
+				if h.Phase == string(delivery.PhaseFailed) {
+					t.Errorf("a transient failure was written failed: %+v", h)
+				}
+			}
+		})
 	}
-	// The want is written again: the retry.
-	b.mu.Lock()
-	b.failWith = ""
-	b.mu.Unlock()
-	b.wants.put(wk, wraw)
-	eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
-	eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
 }
 
 // R6-A AC5: a data module that ran before asks its backup before any resource changes; a refusal holds it, nothing is