Make the switch between composers safe: a composer tells only its own run's agent, a pass whose run ended refuses nothing, a bus that pauses is tried again, a mark keeps what runs and clears at the next start; pin mesh-sdk 7359d18 (issue 1126)

Live 2026-10-28: the own run's last pass, failing on its run's end, marked every module of the controller-mode agent that replaced it (ace 60, shanks 63), and the marks dropped the build and digest because the have read failed in the bus's reload pause. Now: the refusal hook is bound to the run's agent and ctx; a pass whose context ended returns nothing; JetStream not enabled or no responders are tried again after 1, 2 and 4 s before any mark; NotComposed reads the have again while the bus does not answer and marks nothing it cannot read; baseline takes a have a composer marked as what runs, so a mark it did not make clears; each run's shadow lives with the run.
2026-10-28 00:34:10 +01:00
parent 199f6b48b1
commit 4993f1b1d7
9 changed files with 302 additions and 18 deletions
@@ -314,6 +314,44 @@ type notComposedMark struct {
 	prior Have
 }

+// NotComposedDetail opens the detail of a have NotComposed marks: the module keeps running what its build and digest
+// name, and only the node's composition of it failed.
+const NotComposedDetail = "the node cannot compose it, and keeps running what it ran: "
+
+// HavesReadErr is what a Haves may answer besides Read: a have absent (false, nil) told from one that could not be read
+// (false, the error).
+type HavesReadErr interface {
+	ReadErr(ctx context.Context, module string) (Have, bool, error)
+}
+
+// readHave reads a module's have: found, absent, or unread. A read that fails is tried again after a short wait while
+// the bus does not answer (a reload pauses it for a few seconds, issue 1126), and is unread when it still fails. A
+// Haves that cannot tell an absent key from a failed read says absent.
+func (a *Agent) readHave(ctx context.Context, module string) (h Have, found, unread bool) {
+	re, tells := a.env.Haves.(HavesReadErr)
+	if !tells {
+		h, found = a.env.Haves.Read(ctx, module)
+		return h, found, false
+	}
+	for try := 0; ; try++ {
+		h, found, err := re.ReadErr(ctx, module)
+		if err == nil {
+			return h, found, false
+		}
+		if try == 2 || ctx.Err() != nil {
+			return Have{}, false, true
+		}
+		select {
+		case <-ctx.Done():
+			return Have{}, false, true
+		case <-time.After(time.Duration(1<<try) * readAgain):
+		}
+	}
+}
+
+// readAgain is the first wait before a have is read again.
+var readAgain = time.Second
+
 // NotComposed says the node composing its own could not compose its modules (ADR 0547 §2: a stand-in refused, an
 // input that cannot be read): each module with a want gets its have failed, naming why, while it keeps running what it
 // ran, so the failure is seen where every have is read and not only in the journal. The have keeps what runs: the
@@ -364,8 +402,15 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
 				if mark.prior.Phase == "" {
 					mark.prior.Phase = string(delivery.PhaseRunning)
 				}
-			} else if h, ok := a.env.Haves.Read(ctx, module); ok {
+			} else if h, found, unread := a.readHave(ctx, module); found {
 				mark.prior = h // what its last have said runs: this agent has run nothing of it since its start
+			} else if unread {
+				// What its have names cannot be read (the bus paused, issue 1126): a mark written now would drop the build
+				// and digest it names. Nothing is written; the next refusal marks it once the have reads.
+				a.mu.Lock()
+				a.sayOnce("unmarked:"+module, fmt.Sprintf("%s's have cannot be read, so it is not marked: the node cannot compose it: %s", module, why))
+				a.mu.Unlock()
+				continue
 			}
 		}
 		mark.why = why
@@ -373,7 +418,7 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
 		a.notComposed[module] = mark
 		a.mu.Unlock()
 		a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
-			Phase: string(delivery.PhaseFailed), Detail: "the node cannot compose it, and keeps running what it ran: " + why})
+			Phase: string(delivery.PhaseFailed), Detail: NotComposedDetail + why})
 	}
 }

@@ -900,8 +945,15 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 			if h.Composed != "" {
 				// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
 				// failed apply or a different composition is applied again.
+				// A have a composer marked failed (NotComposed) names what still runs: taken as running, so a mark this
+				// agent did not make clears once what it names is what the want names (issue 1126, the stale marks after
+				// the seed's revert).
+				marked := h.Phase == string(delivery.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail)
+				if marked {
+					h.Detail = ""
+				}
 				if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
-					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
+					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning) || marked) {
 					a.mu.Lock()
 					a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
 					a.mu.Unlock()
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/json"
+	"fmt"
 	"slices"
 	"strings"
 	"sync"
@@ -72,6 +73,19 @@ type haves struct {
 	stored  map[string]Have
 	log     []Have
 	removed []string
+	// unreadable is each module whose have the bus cannot read now (ReadErr).
+	unreadable map[string]bool
+}
+
+func (h *haves) ReadErr(ctx context.Context, module string) (Have, bool, error) {
+	h.mu.Lock()
+	bad := h.unreadable[module]
+	h.mu.Unlock()
+	if bad {
+		return Have{}, false, fmt.Errorf("nats: no responders available for request")
+	}
+	x, ok := h.Read(ctx, module)
+	return x, ok, nil
 }

 func (h *haves) Read(_ context.Context, module string) (Have, bool) {
@@ -328,6 +342,27 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
 // What the node-engine itself last applied is what runs after a restart, not the kept declaration.
 func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
 	t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
+	// Issue 1126, the stale marks after the seed's revert: a have a composer marked failed names what still runs; the
+	// agent starting on it applies nothing and says it running, so a mark it did not make clears.
+	t.Run("clears a composer's mark", func(t *testing.T) {
+		b := newBed(t)
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
+		wk, wraw := wantFor(t, "avahi", fp, dig, "")
+		b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
+			Detail: NotComposedDetail + "frombus: the credentials of ace: context canceled"}}
+		b.comps.before(ck, craw)
+		b.wants.before(wk, wraw)
+		stop := b.run()
+		defer stop()
+		eventually(t, "the mark was not cleared", func() bool {
+			h := b.haves.last("avahi")
+			return h.Phase == string(delivery.PhaseRunning) && h.FP == fp && h.Composed == dig && h.Detail == ""
+		})
+		if got := b.appliedList(); len(got) != 0 {
+			t.Fatalf("a module whose mark names what the want names was applied: %v", got)
+		}
+	})
 	b := newBed(t)
 	fp := fpOf("a")
 	ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
@@ -1041,6 +1076,26 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 			t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
 		}
 	}
+	// A wanted module whose have cannot be read (the bus paused, issue 1126) is not marked: a mark would drop the build
+	// and digest its have names.
+	readAgain = time.Millisecond
+	b.haves.mu.Lock()
+	b.haves.unreadable = map[string]bool{"unread": true}
+	b.haves.mu.Unlock()
+	want("unread", fpOf("d"), "")
+	want("fresh", fpOf("e"), "") // never ran, its have absent: marked, naming nothing it runs
+	eventually(t, "unread's want was not taken", func() bool {
+		b.ag.mu.Lock()
+		defer b.ag.mu.Unlock()
+		return b.ag.wants["unread"].Present
+	})
+	b.ag.NotComposed(context.Background(), "a stand-in for web's password, again")
+	if h := b.haves.last("unread"); h.Module != "" {
+		t.Errorf("a module whose have could not be read was marked: %+v", h)
+	}
+	if h := b.haves.last("fresh"); h.Phase != string(delivery.PhaseFailed) || h.FP != "" {
+		t.Errorf("a module that never ran, its have absent, is not marked: %+v", h)
+	}
 	b.ag.NotComposed(context.Background(), "")
 	for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
 		if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {