Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)

Live 2026-10-28: the own run's last pass, failing on its run's end, marked
every module of the controller-mode agent that replaced it (ace 60, shanks 63),
and the marks dropped the build and digest because the have read failed in the
bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx;
a pass whose context ended returns nothing; JetStream not enabled or no
responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads
the have again while the bus does not answer and marks nothing it cannot read;
baseline takes a have a composer marked as what runs, so a mark it did not make
clears; each run's shadow lives with the run.

internal/agent/agent.go

@@ -314,6 +314,44 @@ type notComposedMark struct {
 	prior Have
 }
 
+// NotComposedDetail opens the detail of a have NotComposed marks: the module keeps running what its build and digest
+// name, and only the node's composition of it failed.
+const NotComposedDetail = "the node cannot compose it, and keeps running what it ran: "
+
+// HavesReadErr is what a Haves may answer besides Read: a have absent (false, nil) told from one that could not be read
+// (false, the error).
+type HavesReadErr interface {
+	ReadErr(ctx context.Context, module string) (Have, bool, error)
+}
+
+// readHave reads a module's have: found, absent, or unread. A read that fails is tried again after a short wait while
+// the bus does not answer (a reload pauses it for a few seconds, issue 1126), and is unread when it still fails. A
+// Haves that cannot tell an absent key from a failed read says absent.
+func (a *Agent) readHave(ctx context.Context, module string) (h Have, found, unread bool) {
+	re, tells := a.env.Haves.(HavesReadErr)
+	if !tells {
+		h, found = a.env.Haves.Read(ctx, module)
+		return h, found, false
+	}
+	for try := 0; ; try++ {
+		h, found, err := re.ReadErr(ctx, module)
+		if err == nil {
+			return h, found, false
+		}
+		if try == 2 || ctx.Err() != nil {
+			return Have{}, false, true
+		}
+		select {
+		case <-ctx.Done():
+			return Have{}, false, true
+		case <-time.After(time.Duration(1<<try) * readAgain):
+		}
+	}
+}
+
+// readAgain is the first wait before a have is read again.
+var readAgain = time.Second
+
 // NotComposed says the node composing its own could not compose its modules (ADR 0547 §2: a stand-in refused, an
 // input that cannot be read): each module with a want gets its have failed, naming why, while it keeps running what it
 // ran, so the failure is seen where every have is read and not only in the journal. The have keeps what runs: the
@@ -364,8 +402,15 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
 				if mark.prior.Phase == "" {
 					mark.prior.Phase = string(delivery.PhaseRunning)
 				}
-			} else if h, ok := a.env.Haves.Read(ctx, module); ok {
+			} else if h, found, unread := a.readHave(ctx, module); found {
 				mark.prior = h // what its last have said runs: this agent has run nothing of it since its start
+			} else if unread {
+				// What its have names cannot be read (the bus paused, issue 1126): a mark written now would drop the build
+				// and digest it names. Nothing is written; the next refusal marks it once the have reads.
+				a.mu.Lock()
+				a.sayOnce("unmarked:"+module, fmt.Sprintf("%s's have cannot be read, so it is not marked: the node cannot compose it: %s", module, why))
+				a.mu.Unlock()
+				continue
 			}
 		}
 		mark.why = why
@@ -373,7 +418,7 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
 		a.notComposed[module] = mark
 		a.mu.Unlock()
 		a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
-			Phase: string(delivery.PhaseFailed), Detail: "the node cannot compose it, and keeps running what it ran: " + why})
+			Phase: string(delivery.PhaseFailed), Detail: NotComposedDetail + why})
 	}
 }
 
@@ -900,8 +945,15 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 			if h.Composed != "" {
 				// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
 				// failed apply or a different composition is applied again.
+				// A have a composer marked failed (NotComposed) names what still runs: taken as running, so a mark this
+				// agent did not make clears once what it names is what the want names (issue 1126, the stale marks after
+				// the seed's revert).
+				marked := h.Phase == string(delivery.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail)
+				if marked {
+					h.Detail = ""
+				}
 				if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
-					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
+					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning) || marked) {
 					a.mu.Lock()
 					a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
 					a.mu.Unlock()

internal/agent/agent_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/json"
+	"fmt"
 	"slices"
 	"strings"
 	"sync"
@@ -72,6 +73,19 @@ type haves struct {
 	stored  map[string]Have
 	log     []Have
 	removed []string
+	// unreadable is each module whose have the bus cannot read now (ReadErr).
+	unreadable map[string]bool
+}
+
+func (h *haves) ReadErr(ctx context.Context, module string) (Have, bool, error) {
+	h.mu.Lock()
+	bad := h.unreadable[module]
+	h.mu.Unlock()
+	if bad {
+		return Have{}, false, fmt.Errorf("nats: no responders available for request")
+	}
+	x, ok := h.Read(ctx, module)
+	return x, ok, nil
 }
 
 func (h *haves) Read(_ context.Context, module string) (Have, bool) {
@@ -328,6 +342,27 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
 // What the node-engine itself last applied is what runs after a restart, not the kept declaration.
 func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
 	t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
+	// Issue 1126, the stale marks after the seed's revert: a have a composer marked failed names what still runs; the
+	// agent starting on it applies nothing and says it running, so a mark it did not make clears.
+	t.Run("clears a composer's mark", func(t *testing.T) {
+		b := newBed(t)
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
+		wk, wraw := wantFor(t, "avahi", fp, dig, "")
+		b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
+			Detail: NotComposedDetail + "frombus: the credentials of ace: context canceled"}}
+		b.comps.before(ck, craw)
+		b.wants.before(wk, wraw)
+		stop := b.run()
+		defer stop()
+		eventually(t, "the mark was not cleared", func() bool {
+			h := b.haves.last("avahi")
+			return h.Phase == string(delivery.PhaseRunning) && h.FP == fp && h.Composed == dig && h.Detail == ""
+		})
+		if got := b.appliedList(); len(got) != 0 {
+			t.Fatalf("a module whose mark names what the want names was applied: %v", got)
+		}
+	})
 	b := newBed(t)
 	fp := fpOf("a")
 	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
@@ -1041,6 +1076,26 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 			t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
 		}
 	}
+	// A wanted module whose have cannot be read (the bus paused, issue 1126) is not marked: a mark would drop the build
+	// and digest its have names.
+	readAgain = time.Millisecond
+	b.haves.mu.Lock()
+	b.haves.unreadable = map[string]bool{"unread": true}
+	b.haves.mu.Unlock()
+	want("unread", fpOf("d"), "")
+	want("fresh", fpOf("e"), "") // never ran, its have absent: marked, naming nothing it runs
+	eventually(t, "unread's want was not taken", func() bool {
+		b.ag.mu.Lock()
+		defer b.ag.mu.Unlock()
+		return b.ag.wants["unread"].Present
+	})
+	b.ag.NotComposed(context.Background(), "a stand-in for web's password, again")
+	if h := b.haves.last("unread"); h.Module != "" {
+		t.Errorf("a module whose have could not be read was marked: %+v", h)
+	}
+	if h := b.haves.last("fresh"); h.Phase != string(delivery.PhaseFailed) || h.FP != "" {
+		t.Errorf("a module that never ran, its have absent, is not marked: %+v", h)
+	}
 	b.ag.NotComposed(context.Background(), "")
 	for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
 		if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {

internal/link/have.go

@@ -811,33 +811,46 @@ func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
 // ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
 // be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
 func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {
+	h, ok, _ := q.ReadHaveErr(ctx, module)
+	return h, ok
+}
+
+// ErrNotLinkedHaves is a have read with no link to the bus open that can bind the have bucket.
+var ErrNotLinkedHaves = errors.New("no link to the bus that binds the have bucket is open")
+
+// ReadHaveErr is ReadHave telling a key absent (false, nil) from one that could not be read (false, the error): a bus
+// that does not answer for a moment is not a module that never ran (issue 1126).
+func (q *Session) ReadHaveErr(ctx context.Context, module string) (delivery.Have, bool, error) {
 	q.init()
 	q.mu.Lock()
 	bus := q.bus
 	q.mu.Unlock()
 	hb, ok := bus.(HaveBus)
 	if !ok || bus == nil {
-		return delivery.Have{}, false
+		return delivery.Have{}, false, ErrNotLinkedHaves
 	}
 	key, err := delivery.HaveKey(q.Membership.Node, module)
 	if err != nil {
-		return delivery.Have{}, false
+		return delivery.Have{}, false, err
 	}
 	read, cancel := context.WithTimeout(ctx, q.timeout())
 	defer cancel()
 	bucket, err := hb.HaveBind(read)
 	if err != nil {
-		return delivery.Have{}, false
+		return delivery.Have{}, false, err
 	}
 	value, _, err := bucket.Get(read, key)
+	if errors.Is(err, ErrHaveAbsent) {
+		return delivery.Have{}, false, nil
+	}
 	if err != nil {
-		return delivery.Have{}, false
+		return delivery.Have{}, false, err
 	}
 	h, err := delivery.DecodeHave(value)
 	if err != nil {
-		return delivery.Have{}, false
+		return delivery.Have{}, false, nil // a have no engine can read names nothing it runs
 	}
-	return h, true
+	return h, true, nil
 }
 
 // nothingJudged is the health of a module a look judged no resource of: ok when it was looked at and the module applied

internal/selfcompose/selfcompose.go

@@ -452,10 +452,29 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 	if c.env.Running != nil {
 		running = c.env.Running()
 	}
-	compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
+	scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
+	compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
+	// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
+	// tried again after a growing wait before anything is marked (issue 1126).
+	for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
+		c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
+		select {
+		case <-ctx.Done():
+		case <-time.After(composeAgain[try]):
+		}
+		if ctx.Err() == nil {
+			compositions, _, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
+		}
+	}
+	if ctx.Err() != nil {
+		// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
+		// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
+		return nil
+	}
 	if err != nil {
-		c.refused(fmt.Sprintf("%v", err))
+		// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
 		c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
+		c.refused(fmt.Sprintf("%v", err))
 		return nil
 	}
 	c.refused("")
@@ -904,3 +923,26 @@ func Placed(ctx context.Context, get func(ctx context.Context, bucket, key strin
 		}
 	}
 }
+
+// composeAgain are the waits before a pass that met a bus not answering is tried again.
+var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
+
+// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
+// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
+// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
+// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
+func Transient(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) {
+		return true
+	}
+	words := err.Error()
+	for _, w := range []string{"jetstream not enabled", "no responders"} {
+		if strings.Contains(words, w) {
+			return true
+		}
+	}
+	return false
+}

internal/selfcompose/selfcompose_test.go

@@ -240,6 +240,72 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		t.Fatalf("handed %v, want %s and %s", got, lemursKey, nodeKey)
 	}
 
+	// Issue 1126, the switch made safe: a bus not answering for a moment (a reload of its accounts) is tried again and
+	// refuses nothing; a pass whose run ended refuses nothing either.
+	composeAgain = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	fresh := func(get func(context.Context, string, string) ([]byte, error)) (*Composer, *[]string) {
+		var refusals []string
+		e := b.env()
+		e.Get = get
+		e.Refused = func(why string) { refusals = append(refusals, why) }
+		c2 := New(e)
+		c.mu.Lock()
+		for k, v := range c.wants {
+			c2.wants[k] = v
+		}
+		for k, v := range c.placements {
+			c2.placements[k] = v
+		}
+		for k, v := range c.records {
+			c2.records[k] = v
+		}
+		for k, v := range c.modules {
+			c2.modules[k] = v
+		}
+		c.mu.Unlock()
+		return c2, &refusals
+	}
+	for name, row := range map[string]struct {
+		failures int // reads failing before the bus answers
+		cancel   bool
+		composes bool
+	}{
+		"the bus paused for one read":   {failures: 1, composes: true},
+		"the bus paused past every try": {failures: 100},
+		"the run ended mid-pass":        {failures: 100, cancel: true},
+	} {
+		failed := 0
+		pctx, pcancel := context.WithCancel(ctx)
+		c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
+			if failed < row.failures {
+				failed++
+				if row.cancel {
+					pcancel()
+					return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
+				}
+				return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
+			}
+			b.mu.Lock()
+			defer b.mu.Unlock()
+			return b.kv[bucket+"/"+key], nil
+		})
+		handed := c2.compose(pctx)
+		pcancel()
+		switch {
+		case row.composes && (len(handed) == 0 || len(*refusals) != 0):
+			t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
+		case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
+			t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
+		case row.cancel && len(*refusals) != 0:
+			t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
+		}
+	}
+	if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
+		!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
+		Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
+		t.Error("Transient misjudges a bus that does not answer, or a refusal")
+	}
+
 	// A config value of another module, and a value changing nothing composed, hand nothing.
 	b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
 	quiet(t, out)