ENG-8, ENG-9, ENG-11: a failed pull from a store is tried again once per settled of the store's holder, heard on every node, and the failed have says whom it waits on and what triggered the last retry (design 51, issue 1966; needs mesh-sdk request R1, R2)

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 11:49:44 +01:00
parent 3eef84d979
commit 2a71a8ff76
8 changed files with 467 additions and 17 deletions
@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"

@@ -115,10 +116,14 @@ type bed struct {
 	// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
 	transient bool
 	failTimes int
-	backup    func(module string) Backup
-	kept      map[string][]json.RawMessage
-	others    [][]delivery.Composed
-	said      []string
+	// pullsFn, when set, is the images a failed apply could not pull (called as the apply fails); during runs at the start of
+	// each apply, before it answers.
+	pullsFn func() []PullFailure
+	during  func()
+	backup  func(module string) Backup
+	kept    map[string][]json.RawMessage
+	others  [][]delivery.Composed
+	said    []string
 	// settled is each module Settled was told, and settledRan whether what runs held it then.
 	settled    []string
 	settledRan []bool
@@ -130,6 +135,9 @@ func newBed(t *testing.T) *bed {
 	b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
 	b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
 		Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
+			if b.during != nil {
+				b.during()
+			}
 			b.mu.Lock()
 			defer b.mu.Unlock()
 			b.others = append(b.others, others)
@@ -139,7 +147,11 @@ func newBed(t *testing.T) *bed {
 			}
 			b.applied = append(b.applied, name)
 			if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
-				return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
+				res := Result{Failed: true, Transient: b.transient, Detail: b.failWith}
+				if b.pullsFn != nil {
+					res.Pulls = b.pullsFn()
+				}
+				return res
 			}
 			return Result{}
 		},
@@ -461,6 +473,106 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
 			t.Errorf("a module running its want was applied again (%d applies)", n)
 		}
 	})
+	// design 51 (ENG-8, ENG-11): a pull from a store that failed is tried again when the store's holder settles, once per
+	// settled, never on a clock; the have says whom it waits on and what triggered the last retry.
+	t.Run("a pull from a store waits for its holder to settle", func(t *testing.T) {
+		b := newBed(t)
+		var clock atomic.Int64
+		now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
+		tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
+		b.env.Now = now
+		b.env.Backoff = func(int) time.Duration { return time.Millisecond }
+		b.failWith = "pulling image reg/x@sha256:1: manifest unknown"
+		b.pullsFn = func() []PullFailure {
+			return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: tick()},
+				{Image: "public/y@sha256:2"}}
+		}
+		stop := b.run()
+		defer stop()
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+		wk, wraw := wantFor(t, "web", fp, dig, "")
+		b.comps.put(ck, craw)
+		b.wants.put(wk, wraw)
+		eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		eventually(t, "the have does not say whom it waits on", func() bool {
+			h := b.haves.last("web")
+			return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "tried again once distribution has settled")
+		})
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 1 {
+			t.Fatalf("a failed pull was tried again by the clock (%d applies)", n)
+		}
+		// A settled of another module starts nothing.
+		b.ag.Settled("gitea")
+		time.Sleep(50 * time.Millisecond)
+		if n := len(b.appliedList()); n != 1 {
+			t.Fatalf("a settled of another module applied the module (%d applies)", n)
+		}
+		// The holder's settled: tried again once, whose pull fails again after it (started later): failed again, no more.
+		tick()
+		b.ag.Settled("distribution")
+		eventually(t, "not tried again on the holder's settled", func() bool { return len(b.appliedList()) == 2 })
+		eventually(t, "the have does not say what triggered the retry", func() bool {
+			h := b.haves.last("web")
+			return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "last tried again at") && strings.Contains(h.Detail, "distribution settled")
+		})
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 2 {
+			t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n)
+		}
+		// The next settled answers once more, and the pull works: the module runs, and waits on nothing.
+		b.mu.Lock()
+		b.failWith = ""
+		b.mu.Unlock()
+		tick()
+		b.ag.Settled("distribution")
+		eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 })
+		eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
+		b.ag.Settled("distribution")
+		time.Sleep(50 * time.Millisecond)
+		if n := len(b.appliedList()); n != 3 {
+			t.Fatalf("a running module was applied again by a settled (%d applies)", n)
+		}
+	})
+	// The race just before `settled`: the holder's settled is heard while the pull is in flight, and the pull fails after it;
+	// applied again at once, once, and a settled heard before the pull began applies nothing at once.
+	t.Run("a settled heard while the pull ran is not missed", func(t *testing.T) {
+		b := newBed(t)
+		var clock atomic.Int64
+		now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
+		tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
+		b.env.Now = now
+		b.failWith = "pulling image reg/x@sha256:1: connection refused"
+		began := tick()
+		first := true
+		b.during = func() {
+			if first {
+				first = false
+				tick()
+				b.ag.Settled("distribution") // the holder settles while this apply's pull is in flight
+			}
+		}
+		b.pullsFn = func() []PullFailure {
+			started := began
+			if len(b.applied) > 1 {
+				started = tick() // the retry's pull begins after the settled
+			}
+			return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: started}}
+		}
+		stop := b.run()
+		defer stop()
+		fp := fpOf("a")
+		ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
+		wk, wraw := wantFor(t, "web", fp, dig, "")
+		b.comps.put(ck, craw)
+		b.wants.put(wk, wraw)
+		eventually(t, "the settled heard during the pull was missed", func() bool { return len(b.appliedList()) == 2 })
+		time.Sleep(100 * time.Millisecond)
+		if n := len(b.appliedList()); n != 2 {
+			t.Fatalf("the race retry failing again started a further apply (%d applies)", n)
+		}
+	})
 	// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
 	// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
 	// retrying meanwhile, never failed.