ENG-8, ENG-9, ENG-11: a failed pull from a store is tried again once per settled of the store's holder, heard on every node, and the failed have says whom it waits on and what triggered the last retry (design 51, issue 1966; needs mesh-sdk request R1, R2)

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig

cmd/mesh-host/backup_event_test.go

@@ -6,10 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/agent"
+	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/link"
 )
 
@@ -85,4 +88,62 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
 			t.Errorf("%s: asked %d times, %v", row.name, asked, err)
 		}
 	}
+
+	// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
+	// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
+	told := &settledRecorder{}
+	events := make(chan []byte, 4)
+	for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
+		events <- []byte(body)
+	}
+	linked, end := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
+	deadline := time.Now().Add(2 * time.Second)
+	for len(told.heard()) < 2 && time.Now().Before(deadline) {
+		time.Sleep(5 * time.Millisecond)
+	}
+	end()
+	<-done
+	if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
+		t.Errorf("the agent was told %q settled, want distribution,gitea", got)
+	}
+	// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
+	// one (ENG-8, ENG-10).
+	d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
+		`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
+		`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pull := func(image string) error {
+		return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
+	}
+	if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
+		t.Errorf("a store's image is said as %+v", got)
+	}
+	if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
+		t.Errorf("a public image is said as %+v", got)
+	}
+	if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
+		t.Errorf("a failure that is no pull is said as %+v", got)
+	}
+}
+
+// settledRecorder is the agent a hearing tells.
+type settledRecorder struct {
+	mu   sync.Mutex
+	list []string
+}
+
+func (r *settledRecorder) Settled(holder string) {
+	r.mu.Lock()
+	r.list = append(r.list, holder)
+	r.mu.Unlock()
+}
+
+func (r *settledRecorder) heard() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.list...)
 }

cmd/mesh-host/main.go

@@ -1064,6 +1064,14 @@ func runLink(ctx context.Context, opts options) error {
 	// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
 	trust.onChange(func() { factsOfNow(aside, current.get()) })
 	go followTrust(aside, queue, trust, say)
+	// And the status view's `settled`, for a failed pull from a store to be tried again once the store's holder has settled
+	// (design 51, ENG-11).
+	go followSettled(aside, queue, func() settledTold {
+		if ag := current.get(); ag != nil {
+			return ag
+		}
+		return nil
+	}, say)
 	// And the login it dials as, which the vault's node-logins names for this node (novox/hq ADR 0583 §1).
 	rotator := newRotator(queue, opts.state, opts.timeout, say)
 	go followLogins(aside, queue, mine.Node, func(nl delivery.NodeLogin) {

cmd/mesh-host/nodeagent.go

@@ -731,12 +731,19 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
 	if got.Refused != "" {
 		return failedWith(got.Refused)
 	}
+	// An image that could not be pulled from a store is said with its holder, for the engine to try the apply again once the
+	// holder has settled (design 51, ENG-8).
+	failedOn := func(why string) agent.Result {
+		res := failedWith(why)
+		res.Pulls = pullsOf(d, got.Err)
+		return res
+	}
 	results := apply.ModuleResults(d, got.Outcome, got.Err)
 	if r, ok := results[name]; ok && r.Failed {
-		return failedWith(r.Detail)
+		return failedOn(r.Detail)
 	}
 	if got.Err != nil {
-		return failedWith(got.Err.Error())
+		return failedOn(got.Err.Error())
 	}
 	if got.Probe != nil {
 		return failedWith(got.Probe.partial().Error())

cmd/mesh-host/pulls.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+
+	"github.com/novox/mesh-host/internal/agent"
+	"github.com/novox/mesh-host/internal/apply"
+)
+
+// pullsOf is the images an apply could not find or pull, each with the seat it came through and that seat's holder as the
+// declaration names them beside the container (design 51 *distribution's first delivery*, ENG-8, ENG-10). An image the
+// declaration names no holder for (a public registry's) carries none: its pull is not tried again on a store settling.
+func pullsOf(d *declaration.Declaration, err error) []agent.PullFailure {
+	var out []agent.PullFailure
+	for _, p := range apply.PullErrors(err) {
+		f := agent.PullFailure{Image: p.Image, Started: p.Started}
+		for _, r := range d.Resources {
+			if c, ok := r.(*declaration.Container); ok && c.Image == p.Image && c.ImageHolder != "" {
+				f.Seat, f.Holder = c.ImageSeat, c.ImageHolder
+				break
+			}
+		}
+		out = append(out, f)
+	}
+	return out
+}

cmd/mesh-host/settled.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// settledEvent is where the status view says a module has settled on a node: its have reached its want, running and healthy
+// (design 47; the view's `settled`, said once for a module on a node at one composition). The engine hears it for every module
+// and tells its agent, which tries again, once per settled, a failed apply that waits on that module as the holder of the store
+// its image comes from (design 51 *distribution's first delivery*, issue 1966 ENG-11). The grants part grants the engine this
+// subject (NodeAgentHears); a refused subscription is said, and made again when the next link opens.
+const settledEvent = "mesh.mod.mesh-delivery-view.event.settled"
+
+// settledBody is what of the event the engine reads: the module that settled, on whichever node.
+type settledBody struct {
+	Module string `json:"module"`
+}
+
+// settledTold is who is told a holder has settled.
+type settledTold interface{ Settled(holder string) }
+
+// followSettled hears the status view's `settled` on every link, never on a timer, until ctx ends. now names the agent running
+// now (nil while the node agent begins again): a settled heard while there is none is not lost for it, since a new agent applies
+// again every have that failed at its start.
+func followSettled(ctx context.Context, session *link.Session, now func() settledTold, say link.Announce) {
+	said := ""
+	for ctx.Err() == nil {
+		linked, cancel, err := session.WhenLinked(ctx)
+		if err != nil {
+			return
+		}
+		events, stop, herr := session.Hear(settledEvent)
+		if herr != nil {
+			if line := fmt.Sprintf("a store settling is not heard yet (%s: %v): a failed pull waits for the next link, or for compose", settledEvent, herr); line != said {
+				said = line
+				say(line)
+			}
+		} else {
+			said = ""
+			listen(linked, events, now, say)
+			stop()
+		}
+		<-linked.Done()
+		cancel()
+	}
+}
+
+// listen tells the agent each settled it hears, until the link ends.
+func listen(linked context.Context, events <-chan []byte, now func() settledTold, say link.Announce) {
+	for {
+		select {
+		case <-linked.Done():
+			return
+		case raw, ok := <-events:
+			if !ok {
+				return
+			}
+			var b settledBody
+			if json.Unmarshal(raw, &b) != nil || b.Module == "" {
+				continue
+			}
+			if told := now(); told != nil {
+				told.Settled(b.Module)
+			}
+		}
+	}
+}

internal/agent/agent.go

@@ -82,6 +82,9 @@ type Haves interface {
 
 // Result is how one apply or removal came out.
 type Result struct {
+	// Pulls are the images the apply could not find or pull, each with the seat it came through and that seat's holder
+	// where the declaration names them (a store's image, ENG-10): what a failed apply waits on (ENG-8).
+	Pulls  []PullFailure
 	Failed bool
 	// Transient says the failure was the bus not answering for a moment (no responders, a timeout, a reconnect, a
 	// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
@@ -89,6 +92,14 @@ type Result struct {
 	Detail    string
 }
 
+// PullFailure is one image an apply could not find or pull. Holder is the module holding the seat the image came through,
+// "" for an image no store holds (a public registry's): its pull is tried again only by the changes that always start an
+// apply. Started is when the pull began.
+type PullFailure struct {
+	Image, Seat, Holder string
+	Started             time.Time
+}
+
 // Backup is the answer of the backup asked before a module moves.
 // BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
 // the next start backs the module up again rather than holding it for the operator.
@@ -150,6 +161,9 @@ type Env struct {
 	// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
 	// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
 	RunsChanged func(why string)
+	// Now is the clock the agent compares a pull's start with the time it heard a store's holder settled by; nil is
+	// time.Now.
+	Now func() time.Time
 	// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
 	// transient failure only on RetryTransient or a change of either key.
 	Backoff func(try int) time.Duration
@@ -232,11 +246,17 @@ type Agent struct {
 	failedAgain chan string
 	// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
 	// transient asks for every transiently failed module at once. stopped is closed when Run returns.
-	tries     map[string]int
-	gen       uint64
-	again     chan againAsk
-	transient chan struct{}
-	stopped   chan struct{}
+	tries map[string]int
+	// pulls is, for each module whose last apply failed on a pull from a store, what it waits on (ENG-8); settledAt and
+	// settledSeq are, by the module holding a store, when its settled was last heard since this agent started and how many
+	// have been. Under mu.
+	pulls      map[string]*pullWait
+	settledAt  map[string]time.Time
+	settledSeq map[string]int
+	gen        uint64
+	again      chan againAsk
+	transient  chan struct{}
+	stopped    chan struct{}
 }
 
 type againAsk struct {
@@ -276,6 +296,7 @@ func New(env Env) *Agent {
 	}
 	return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
 		attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
+		pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
 		retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
 		transient: make(chan struct{}, 1), stopped: make(chan struct{})}
 }
@@ -1510,6 +1531,9 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 	a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
 	res := a.env.Apply(ctx, comp.C, others)
 	a.mu.Lock()
+	if !res.Failed || res.Transient {
+		delete(a.pulls, module)
+	}
 	what := fmt.Sprintf("%s at %s", module, fp)
 	if module == NodeModule {
 		what = "the node's own composition"
@@ -1544,11 +1568,20 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 	delete(a.tries, module)
 	if res.Failed {
 		a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
+		detail := res.Detail
+		again := a.notePullFailed(module, res.Pulls)
+		if words := a.pullWords(module); words != "" {
+			detail += words
+		}
 		a.mu.Unlock()
 		a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
-		a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
+		a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail})
+		if again != "" {
+			a.askPullAgain(module, again)
+		}
 		return
 	}
+	delete(a.pulls, module)
 	delete(a.attempt, module)
 	delete(a.notComposed, module) // the apply says what it runs now
 	a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
@@ -1584,6 +1617,7 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
 		return
 	}
 	delete(a.attempt, module)
+	delete(a.pulls, module)
 	delete(a.had, module)
 	a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
 	a.mu.Unlock()
@@ -1794,3 +1828,107 @@ func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
 		}
 	}
 }
+
+// pullWait is what a module whose apply failed on a pull from a store waits on: the store's holder settling (ENG-8,
+// design 51 *distribution's first delivery*).
+type pullWait struct {
+	PullFailure
+	// retriedFor is the count of the holder's settled the last retry answered: a settled answers once.
+	retriedFor int
+	// last is what triggered the last retry, in words, and lastAt when; the have and the journal say it.
+	last   string
+	lastAt time.Time
+}
+
+func (a *Agent) now() time.Time {
+	if a.env.Now != nil {
+		return a.env.Now()
+	}
+	return time.Now()
+}
+
+// notePullFailed records the pull a failed apply met from a store (the first image with a holder), keeping the settled it was
+// last tried again for. It answers, when the holder's settled was heard after the pull started and not yet answered, the
+// words of the retry to make at once (the race just before `settled`: ENG-11), else "". a.mu is held.
+func (a *Agent) notePullFailed(module string, pulls []PullFailure) string {
+	var found *PullFailure
+	for i := range pulls {
+		if pulls[i].Holder != "" {
+			found = &pulls[i]
+			break
+		}
+	}
+	if found == nil {
+		delete(a.pulls, module)
+		return ""
+	}
+	pw := &pullWait{PullFailure: *found}
+	if before := a.pulls[module]; before != nil {
+		pw.retriedFor, pw.last, pw.lastAt = before.retriedFor, before.last, before.lastAt
+	}
+	a.pulls[module] = pw
+	holder := found.Holder
+	if seq := a.settledSeq[holder]; seq > pw.retriedFor && a.settledAt[holder].After(found.Started) {
+		return a.markPullRetry(pw, seq, fmt.Sprintf("%s settled at %s, after the pull of %s began at %s", holder,
+			a.settledAt[holder].Format("15:04:05"), found.Image, found.Started.Format("15:04:05")))
+	}
+	return ""
+}
+
+// markPullRetry notes that pw is tried again for the holder's settled number seq, and answers the words that say so. a.mu
+// is held.
+func (a *Agent) markPullRetry(pw *pullWait, seq int, why string) string {
+	pw.retriedFor, pw.last, pw.lastAt = seq, why, a.now()
+	return why
+}
+
+// pullWords is what a failed have says of the wait: whose settling it is tried again on and what triggered the last retry.
+// "" for a failure that waits on no holder. a.mu is held.
+func (a *Agent) pullWords(module string) string {
+	pw := a.pulls[module]
+	if pw == nil {
+		return ""
+	}
+	words := fmt.Sprintf("; it is tried again once %s has settled", pw.Holder)
+	if pw.last != "" {
+		words += "; last tried again at " + pw.lastAt.Format("15:04:05") + ": " + pw.last
+	}
+	return words
+}
+
+// askPullAgain asks the apply of a module again, because why, as a person's verb compose does (RetryFailed): said in the
+// journal, never blocking.
+func (a *Agent) askPullAgain(module, why string) {
+	select {
+	case a.failedAgain <- module:
+		a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store; applied again because %s", module, why))
+	default:
+		a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store, and %s, but the engine has too many retries "+
+			"asked; ask it with compose", module, why))
+	}
+}
+
+// Settled is the status view's word that a module has settled on some node (ENG-11): the engine's hearing tells it for every
+// module it hears. When the module holds a store a failed apply of this node waits on, that apply is tried again once for
+// this settled; a settled of any other module starts nothing. A pull that began after this settled, or has failed before it
+// and is answered, is not tried again for it (notePullFailed).
+func (a *Agent) Settled(holder string) {
+	a.mu.Lock()
+	a.settledSeq[holder]++
+	a.settledAt[holder] = a.now()
+	seq := a.settledSeq[holder]
+	var asks []string
+	var modules []string
+	for module, pw := range a.pulls {
+		att, failed := a.attempt[module]
+		if pw.Holder != holder || !failed || att.Held || att.Transient || pw.retriedFor >= seq {
+			continue
+		}
+		modules = append(modules, module)
+		asks = append(asks, a.markPullRetry(pw, seq, fmt.Sprintf("%s settled", holder)))
+	}
+	a.mu.Unlock()
+	for i, module := range modules {
+		a.askPullAgain(module, asks[i])
+	}
+}

internal/agent/agent_test.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -115,10 +116,14 @@ type bed struct {
 	// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
 	transient bool
 	failTimes int
-	backup    func(module string) Backup
-	kept      map[string][]json.RawMessage
-	others    [][]delivery.Composed
-	said      []string
+	// pullsFn, when set, is the images a failed apply could not pull (called as the apply fails); during runs at the start of
+	// each apply, before it answers.
+	pullsFn func() []PullFailure
+	during  func()
+	backup  func(module string) Backup
+	kept    map[string][]json.RawMessage
+	others  [][]delivery.Composed
+	said    []string
 	// settled is each module Settled was told, and settledRan whether what runs held it then.
 	settled    []string
 	settledRan []bool
@@ -130,6 +135,9 @@ func newBed(t *testing.T) *bed {
 	b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
 	b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
 		Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
+			if b.during != nil {
+				b.during()
+			}
 			b.mu.Lock()
 			defer b.mu.Unlock()
 			b.others = append(b.others, others)
@@ -139,7 +147,11 @@ func newBed(t *testing.T) *bed {
 			}
 			b.applied = append(b.applied, name)
 			if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
-				return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
+				res := Result{Failed: true, Transient: b.transient, Detail: b.failWith}
+				if b.pullsFn != nil {
+					res.Pulls = b.pullsFn()
+				}
+				return res
 			}
 			return Result{}
 		},
@@ -461,6 +473,106 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
 			t.Errorf("a module running its want was applied again (%d applies)", n)
 		}
 	})
+	// design 51 (ENG-8, ENG-11): a pull from a store that failed is tried again when the store's holder settles, once per
+	// settled, never on a clock; the have says whom it waits on and what triggered the last retry.
+	t.Run("a pull from a store waits for its holder to settle", func(t *testing.T) {
+		b := newBed(t)
+		var clock atomic.Int64
+		now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
+		tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
+		b.env.Now = now
+		b.env.Backoff = func(int) time.Duration { return time.Millisecond }
+		b.failWith = "pulling image reg/x@sha256:1: manifest unknown"
+		b.pullsFn = func() []PullFailure {
+			return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: tick()},
+				{Image: "public/y@sha256:2"}}
+		}
+		stop := b.run()
+		defer stop()
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+		wk, wraw := wantFor(t, "web", fp, dig, "")
+		b.comps.put(ck, craw)
+		b.wants.put(wk, wraw)
+		eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		eventually(t, "the have does not say whom it waits on", func() bool {
+			h := b.haves.last("web")
+			return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "tried again once distribution has settled")
+		})
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 1 {
+			t.Fatalf("a failed pull was tried again by the clock (%d applies)", n)
+		}
+		// A settled of another module starts nothing.
+		b.ag.Settled("gitea")
+		time.Sleep(50 * time.Millisecond)
+		if n := len(b.appliedList()); n != 1 {
+			t.Fatalf("a settled of another module applied the module (%d applies)", n)
+		}
+		// The holder's settled: tried again once, whose pull fails again after it (started later): failed again, no more.
+		tick()
+		b.ag.Settled("distribution")
+		eventually(t, "not tried again on the holder's settled", func() bool { return len(b.appliedList()) == 2 })
+		eventually(t, "the have does not say what triggered the retry", func() bool {
+			h := b.haves.last("web")
+			return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "last tried again at") && strings.Contains(h.Detail, "distribution settled")
+		})
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 2 {
+			t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n)
+		}
+		// The next settled answers once more, and the pull works: the module runs, and waits on nothing.
+		b.mu.Lock()
+		b.failWith = ""
+		b.mu.Unlock()
+		tick()
+		b.ag.Settled("distribution")
+		eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 })
+		eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
+		b.ag.Settled("distribution")
+		time.Sleep(50 * time.Millisecond)
+		if n := len(b.appliedList()); n != 3 {
+			t.Fatalf("a running module was applied again by a settled (%d applies)", n)
+		}
+	})
+	// The race just before `settled`: the holder's settled is heard while the pull is in flight, and the pull fails after it;
+	// applied again at once, once, and a settled heard before the pull began applies nothing at once.
+	t.Run("a settled heard while the pull ran is not missed", func(t *testing.T) {
+		b := newBed(t)
+		var clock atomic.Int64
+		now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
+		tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
+		b.env.Now = now
+		b.failWith = "pulling image reg/x@sha256:1: connection refused"
+		began := tick()
+		first := true
+		b.during = func() {
+			if first {
+				first = false
+				tick()
+				b.ag.Settled("distribution") // the holder settles while this apply's pull is in flight
+			}
+		}
+		b.pullsFn = func() []PullFailure {
+			started := began
+			if len(b.applied) > 1 {
+				started = tick() // the retry's pull begins after the settled
+			}
+			return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: started}}
+		}
+		stop := b.run()
+		defer stop()
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+		wk, wraw := wantFor(t, "web", fp, dig, "")
+		b.comps.put(ck, craw)
+		b.wants.put(wk, wraw)
+		eventually(t, "the settled heard during the pull was missed", func() bool { return len(b.appliedList()) == 2 })
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 2 {
+			t.Fatalf("the race retry failing again started a further apply (%d applies)", n)
+		}
+	})
 	// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
 	// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
 	// retrying meanwhile, never failed.

internal/apply/apply.go

@@ -3481,16 +3481,17 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error {
 	// already hold — built here, or handed over. There is no registry that answers for it, so
 	// pulling would fail somewhere that names a network problem instead of a missing image.
 	if strings.HasPrefix(image, "sha256:") {
-		return &PullError{Image: image, Err: fmt.Errorf(
+		return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf(
 			"image %s is not on this machine, and an image named by its own digest cannot be "+
 				"fetched: nothing serves it. Build it here, or load it, before applying this",
 			image)}
 	}
+	started := time.Now()
 	if _, err := run(ctx, cri, "pull", image); err != nil {
-		return &PullError{Image: image, Err: fmt.Errorf("pulling image %s: %w", image, err)}
+		return &PullError{Image: image, Started: started, Err: fmt.Errorf("pulling image %s: %w", image, err)}
 	}
 	if _, err := run(ctx, cri, "image", "inspect", image); err != nil {
-		return &PullError{Image: image, Err: fmt.Errorf("image %s is not present after pulling it: %w", image, err)}
+		return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf("image %s is not present after pulling it: %w", image, err)}
 	}
 	return nil
 }
@@ -4405,7 +4406,33 @@ func pullFirst(ctx context.Context, ordered []declaration.Resource, d *declarati
 // pull from another failed apply (ENG-8).
 type PullError struct {
 	Image string
-	Err   error
+	// Started is when the pull began, which the node agent compares with when a store's holder settled (ENG-11).
+	Started time.Time
+	Err     error
+}
+
+// PullErrors are the pull errors of every failure an apply reports, the first among them and the rest in Failures, in the
+// order they happened: what the node agent tells a failed pull from another failed apply by (ENG-8). Nil for an error
+// that is no apply's, or whose failures met none.
+func PullErrors(err error) []*PullError {
+	var out []*PullError
+	var e *Error
+	if errors.As(err, &e) {
+		for _, f := range e.Failures {
+			var p *PullError
+			if errors.As(f.Err, &p) {
+				out = append(out, p)
+			}
+		}
+		if len(out) > 0 {
+			return out
+		}
+	}
+	var p *PullError
+	if errors.As(err, &p) {
+		out = append(out, p)
+	}
+	return out
 }
 
 func (e *PullError) Error() string { return e.Err.Error() }