Retry an apply, a fetch by digest and a have write that met the bus reloading, and keep the cause in the have

The bus node's engine writes the bus user list (ADR 0547 bridge); each write makes the bus reload its accounts, and
for about three seconds every JetStream request meets no responders. An apply that failed then was kept failed until a
key changed (mesh-vault on novox, 2026-10-25 23:04:24), holding every module that needs the secret seat. A transient
bus failure is now retried with a backoff (2 s, 5 s, 15 s, 60 s, then every 5 min), the have saying retrying meanwhile;
after the engine writes the user list, the transiently failed modules are tried again once the reload is over. A have
write that met it is written again the same way. shorten keeps the head and the tail, the cause.

cmd/mesh-host/file_references_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -9,13 +10,20 @@ import (
 	"testing"
 
 	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-host/internal/link"
 )
 
-// aFilesLink is a link that serves objects by name.
-type aFilesLink struct{ objects map[string][]byte }
+// aFilesLink is a link that serves objects by name, or fails every read with err.
+type aFilesLink struct {
+	objects map[string][]byte
+	err     error
+}
 
 func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
+	if a.err != nil {
+		return nil, a.err
+	}
 	if b, ok := a.objects[name]; ok {
 		return b, nil
 	}
@@ -40,7 +48,7 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
 
 	bad := append([]byte(nil), body...)
 	bad[10] ^= 1
-	ctx := link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: bad}})
+	ctx := link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: bad}})
 	report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
 	if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
 		t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
@@ -48,8 +56,24 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
 	if _, err := os.Stat(target); err == nil {
 		t.Error("1695/AC2: a file was written from an object that differs")
 	}
+	if failedWith(report.Refused).Transient {
+		t.Errorf("an object that differs is a real failure, and was taken as the bus not answering: %s", report.Refused)
+	}
 
-	ctx = link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: body}})
+	// The bus answers no JetStream request while it reloads its accounts (2026-10-25 23:04:24, mesh-vault on the bus
+	// node): the failure is transient, so the agent tries it again, and the detail kept is its cause, the tail.
+	noResponders := fmt.Errorf("fetching the object 2bd98ebd of this node's own object bucket: reading the chunks of "+
+		"object %s: %w", strings.Repeat("x", 300), nats.ErrNoResponders)
+	ctx = link.WithFiles(context.Background(), aFilesLink{err: noResponders})
+	report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
+	failed := failedWith("a file of this composition could not be had by its digest: file mesh-vault.kept: " + report.Refused)
+	if !failed.Transient || !strings.HasSuffix(failed.Detail, nats.ErrNoResponders.Error()) || len(failed.Detail) > 310 ||
+		!strings.HasPrefix(failed.Detail, "a file of this composition") {
+		t.Errorf("a fetch that met no responders: transient %v, detail %q (%d bytes); want transient, its head and its cause",
+			failed.Transient, failed.Detail, len(failed.Detail))
+	}
+
+	ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: body}})
 	report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
 	if report.Refused != "" {
 		t.Fatalf("1695/AC1: refused: %s", report.Refused)

cmd/mesh-host/nodeagent.go

@@ -80,13 +80,23 @@ func (h sessionHaves) Removed(ctx context.Context, module string) {
 	h.session.NoteRemoved(ctx, module, "")
 }
 
-// shorten keeps a have's detail within what a have value holds.
+// shorten keeps a have's detail within what a have value holds: its head, which says what failed, and its tail, which is
+// the cause (an error wrapped five times names the bus's "no responders" only at its end: 2026-10-25 23:04, mesh-vault on
+// the bus node said only which object it read).
 func shorten(s string) string {
-	const most = 300
+	const most, head = 300, 100
 	if len(s) <= most {
 		return s
 	}
-	return s[:most-1] + "…"
+	const gap = " … "
+	tail := s[len(s)-(most-head-len(gap)):]
+	return strings.ToValidUTF8(s[:head], "") + gap + strings.ToValidUTF8(tail, "")
+}
+
+// failedWith is a failed apply that says why, transient when the words are the bus not answering for a moment
+// (link.TransientWords): the agent then tries it again without a new revision.
+func failedWith(why string) agent.Result {
+	return agent.Result{Failed: true, Transient: link.TransientWords(why), Detail: shorten(why)}
 }
 
 // backupWithin is how long a backup is given from its ask to say it is done (ADR 0429 §14).
@@ -393,7 +403,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 		kept := keptOnce(opts, signer)
 		// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
 		// node, and a node whose compositions carry no list writes nothing.
-		users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say)
+		users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
+			// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
+			// seconds: what failed in them is tried again once that is over, not at its next wait.
+			time.AfterFunc(reloadSettles, func() {
+				if now := current.get(); now != nil {
+					now.RetryTransient()
+				}
+			})
+		})
 		go func() {
 			if err := users.Run(linked); err != nil && linked.Err() == nil {
 				say("the bus's user list is not followed on this link: " + err.Error())
@@ -425,6 +443,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			// again (ace, 2026-10-25 21:17).
 			Settled: func(module string) { knockFilling(module, say) },
 			Say:     say,
+			// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
+			Backoff: link.Backoff,
 			Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
 				actx, done := applyContext(session.WithLink(ctx))
 				res := applyComposition(actx, opts, c, others, sched, say)
@@ -471,6 +491,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 	}
 }
 
+// reloadSettles is how long after this engine wrote the bus's user list the bus is taken to have reloaded its accounts:
+// the bus module's entrypoint looks at the list every 5 s and then signals the server, whose reload of the accounts
+// answers no JetStream request for about three seconds (on the bus node, 2026-10-25).
+var reloadSettles = 10 * time.Second
+
 // agentNow is the agent running now, which the node's facts are built from.
 type agentNow struct {
 	mu sync.Mutex
@@ -501,25 +526,25 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
 	}
 	d, err := declarationOf(c)
 	if err != nil {
-		return agent.Result{Failed: true, Detail: shorten(err.Error())}
+		return failedWith(err.Error())
 	}
 	if err := refuseEmptyWrites(d); err != nil {
-		return agent.Result{Failed: true, Detail: shorten(err.Error())}
+		return failedWith(err.Error())
 	}
 	scope := apply.Scope{Module: name, Others: resourcesOf(others)}
 	got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
 	if got.Refused != "" {
-		return agent.Result{Failed: true, Detail: shorten(got.Refused)}
+		return failedWith(got.Refused)
 	}
 	results := apply.ModuleResults(d, got.Outcome, got.Err)
 	if r, ok := results[name]; ok && r.Failed {
-		return agent.Result{Failed: true, Detail: shorten(r.Detail)}
+		return failedWith(r.Detail)
 	}
 	if got.Err != nil {
-		return agent.Result{Failed: true, Detail: shorten(got.Err.Error())}
+		return failedWith(got.Err.Error())
 	}
 	if got.Probe != nil {
-		return agent.Result{Failed: true, Detail: shorten(got.Probe.partial().Error())}
+		return failedWith(got.Probe.partial().Error())
 	}
 	return agent.Result{}
 }

cmd/mesh-host/userlist.go

@@ -23,7 +23,7 @@ import (
 // userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
 // agent trusts, the write alone, and the have.
 func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey,
-	say link.Announce) *userlist.Follower {
+	say link.Announce, wrote func()) *userlist.Follower {
 	return userlist.New(userlist.Env{
 		Node:     session.Membership.Node,
 		Composed: bucketWatch{session, composedBucket},
@@ -33,7 +33,11 @@ func userListFollower(opts options, session *link.Session, composedBucket string
 		},
 		OnDisk: userListOnDisk,
 		Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
-			return writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
+			digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
+			if changed && wrote != nil {
+				wrote()
+			}
+			return digest, changed, err
 		},
 		Report: func(ctx context.Context, module string, w userlist.Written) {
 			session.NoteUserList(ctx, module, w)