Pin the group's mesh-sdk head 0d97c09c for the retake class (issue 1748)

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go

@@ -62,6 +62,8 @@ type fakeBuildModules struct {
 	js        jetstream.JetStream
 	ctx       context.Context
 	failBuild atomic.Bool
+	// failRetakeable makes every build fail as a lookup that timed out (delivery.Retake), which passes by itself.
+	failRetakeable atomic.Bool
 	// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
 	refusePush atomic.Bool
 	mu         sync.Mutex
@@ -117,6 +119,8 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any,
 			merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
 		}
 		return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil
+	case f.failRetakeable.Load() && tool != "build_image_copy":
+		return nil, fmt.Errorf("%slookup git.novox.be on 10.10.0.3:53: i/o timeout", delivery.Retake)
 	case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs
 		return nil, fmt.Errorf("go build of %v: boom", args["artifact"])
 	case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
@@ -491,30 +495,47 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
 // failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
 // the key again as a new revision of the same key, now done.
 func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
-	b := startBed(t)
-	b.fake.failBuild.Store(true)
-	b.archive(t, "ca-trust", "modules/ca-trust", commit)
-	b.start(t, "n1", nil)
-	v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
-	if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
-		t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
-	}
-	key, _ := delivery.BuilderBuildKey("ca-trust", fp)
-	failed, err := b.builds.Get(b.ctx, key)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b.fake.failBuild.Store(false)
-	// The indexer writes the module again (here at a later commit), with the same fingerprint.
-	b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
-	b.await(t, "ca-trust", isState(delivery.BuildDone))
-	done, err := b.builds.Get(b.ctx, key)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if done.Revision() <= failed.Revision() {
-		t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
-	}
+	t.Run("a new modules value with the same fingerprint", func(t *testing.T) {
+		b := startBed(t)
+		b.fake.failBuild.Store(true)
+		b.archive(t, "ca-trust", "modules/ca-trust", commit)
+		b.start(t, "n1", nil)
+		v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
+		if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
+			t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
+		}
+		key, _ := delivery.BuilderBuildKey("ca-trust", fp)
+		failed, err := b.builds.Get(b.ctx, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		b.fake.failBuild.Store(false)
+		// The indexer writes the module again (here at a later commit), with the same fingerprint.
+		b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
+		b.await(t, "ca-trust", isState(delivery.BuildDone))
+		done, err := b.builds.Get(b.ctx, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if done.Revision() <= failed.Revision() {
+			t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
+		}
+	})
+	// 1748/AC6: a build failed retakeable is built again after its backoff, with no new modules value or other event, and
+	// ends done; the key says it was retaken.
+	t.Run("a retakeable failure after its backoff", func(t *testing.T) {
+		b := startBed(t)
+		b.fake.failRetakeable.Store(true)
+		b.archive(t, "ca-trust", "modules/ca-trust", commit)
+		b.start(t, "n1", map[string]string{"MESH_BUILDER_RETAKE_BACKOFF": "1s"})
+		if v := b.await(t, "ca-trust", isState(delivery.BuildFailed)); !delivery.Retakeable(v.Error) {
+			t.Fatalf("build = %+v, want failed retakeable", v)
+		}
+		b.fake.failRetakeable.Store(false) // the lookup passes
+		if v := b.await(t, "ca-trust", isState(delivery.BuildDone)); v.Retakes < 1 {
+			t.Errorf("done key = %+v, want its retake counted", v)
+		}
+	})
 }
 
 // TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -77,6 +77,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 			return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
 		}
 	}
+	var retakeBackoff time.Duration // 0 is build.DefaultRetakeBackoff
+	if v := strings.TrimSpace(getenv("MESH_BUILDER_RETAKE_BACKOFF")); v != "" {
+		if retakeBackoff, err = time.ParseDuration(v); err != nil {
+			return fmt.Errorf("MESH_BUILDER_RETAKE_BACKOFF: %w", err)
+		}
+	}
 	tools := &buildrun.ToolchainSet{}
 	// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
 	// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
@@ -121,6 +127,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		Logf:       logger.Printf,
 		Stored:     storedVia(calls.Call),
 		Node:       cred.Node,
+		// A key failed retakeable (delivery.Retake) is taken again by itself after this wait, at most delivery.MaxRetakes times (issue 1748).
+		RetakeBackoff: retakeBackoff,
 	}
 	// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
 	// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision

modules/mesh-delivery-builder/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-builder
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-builder/go.sum

@@ -1,7 +1,7 @@
 git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
 git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4 h1:K2niszrlnJ7JMiXI6xg4gG90YCd/tqNLGBvujLInI48=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-delivery-builder/internal/build/claim.go

@@ -66,6 +66,10 @@ type ClaimOptions struct {
 	// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
 	// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
 	Stored func(ctx context.Context, references []string) (gone []string, err error)
+	// Retaking is this claim being the builder's backoff taking a retakeable failed key again (Builder.armRetake, issue
+	// 1748): only it adds to BuilderBuild.Retakes, so the cap of delivery.MaxRetakes counts the backoff's own retakes and
+	// not a look another event makes while the backoff waits.
+	Retaking bool
 }
 
 // StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store
@@ -153,7 +157,8 @@ func Unreferenced(b delivery.BuilderBuild) bool {
 //   - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
 //     builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
 //     clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
-//     which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
+//     which wait for another fingerprint (a retakeable failure, delivery.Retakeable, is taken again too; the new
+//     revision keeps Retakes, and counts one more only when it is the backoff's claim, ClaimOptions.Retaking), and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
 //     another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
 //   - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
 //     published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
@@ -212,6 +217,10 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
 	case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
 		return nil, nil
 	case cur.State == delivery.BuildFailed:
+		building.Retakes = cur.Retakes // the count rides on the key's value, so it survives a restart (issue 1748)
+		if opt.Retaking && delivery.Retakeable(cur.Error) {
+			building.Retakes++
+		}
 		return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
 	case cur.State == delivery.BuildQueued && cur.Waits != "":
 		if opt.Published == nil || !opt.Published(cur.Waits) {

modules/mesh-delivery-builder/internal/build/retake_test.go

@@ -0,0 +1,179 @@
+package build
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// clock is the builder's fake clock for the retake backoff (issue 1748): After records what is asked and fire runs it.
+type clock struct {
+	mu      sync.Mutex
+	waits   []time.Duration
+	pending []func()
+}
+
+func (c *clock) after(d time.Duration, f func()) func() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.waits, c.pending = append(c.waits, d), append(c.pending, f)
+	return func() {}
+}
+
+// fire runs every timer set so far, once, and answers how many there were.
+func (c *clock) fire() int {
+	c.mu.Lock()
+	fs := c.pending
+	c.pending = nil
+	c.mu.Unlock()
+	for _, f := range fs {
+		f()
+	}
+	return len(fs)
+}
+
+func retakeRig(t *testing.T) (*rig, *clock) {
+	r := newRig(t, goBundle)
+	c := &clock{}
+	r.b.RetakeBackoff, r.b.After = 10*time.Second, c.after
+	return r, c
+}
+
+const lookupFailure = delivery.Retake + "go mod download: lookup git.novox.be on 10.10.0.3:53: i/o timeout"
+
+// 1748: a key failed retakeable is taken again by the builder's backoff, and only by it counts toward the cap.
+func TestTheBuilderRetakesARetakeableFailure(t *testing.T) {
+	// AC3: claimed again as a new revision after the backoff, with no other event; the count kept on the key stops it after
+	// delivery.MaxRetakes, and a builder that starts again reads the count.
+	t.Run("after its backoff, at most three times", func(t *testing.T) {
+		r, c := retakeRig(t)
+		r.bm.fails["ace/build_go_build"] = lookupFailure
+		v := r.consider(t, "modules/m")
+		if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes != 0 {
+			t.Fatalf("build = %+v, want failed retakeable with no retake yet", v)
+		}
+		if len(c.waits) != 1 || c.waits[0] != 10*time.Second {
+			t.Fatalf("backoffs = %v, want one of 10s", c.waits)
+		}
+		rev := r.st.revs[r.key]
+		for retake := 1; retake <= delivery.MaxRetakes; retake++ {
+			if c.fire() != 1 {
+				t.Fatalf("retake %d: no timer was set", retake)
+			}
+			r.b.Wait()
+			v = valueOf(t, r.st, r.key)
+			if v.Retakes != retake || r.st.revs[r.key] <= rev {
+				t.Fatalf("retake %d: key = %+v at revision %d (was %d)", retake, v, r.st.revs[r.key], rev)
+			}
+			rev = r.st.revs[r.key]
+		}
+		if len(c.pending) != 0 || len(c.waits) != delivery.MaxRetakes || c.waits[1] != 20*time.Second || c.waits[2] != 40*time.Second {
+			t.Fatalf("after %d retakes: %d pending, backoffs %v; want none pending and 10s, 20s, 40s", delivery.MaxRetakes, len(c.pending), c.waits)
+		}
+		if n := len(r.bm.callsOf("build_go_build")); n != 1+delivery.MaxRetakes {
+			t.Errorf("%d builds, want the first and %d retakes", n, delivery.MaxRetakes)
+		}
+		// A builder that starts again reads the count from the key: at the cap it arms nothing, below it the next wait.
+		restarted := func(retakes int) *clock {
+			rr, cc := retakeRig(t)
+			rr.st, rr.b.Claimer.Store = r.st, r.st
+			data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildFailed, Error: lookupFailure, Retakes: retakes})
+			if err != nil {
+				t.Fatal(err)
+			}
+			r.st.vals[r.key] = data
+			rr.b.init()
+			rr.b.known["m"] = delivery.Module{FP2: fp1, Manifest: mdigest('a')}
+			if err := rr.b.Sweep(context.Background()); err != nil {
+				t.Fatal(err)
+			}
+			return cc
+		}
+		if cc := restarted(delivery.MaxRetakes); len(cc.waits) != 0 {
+			t.Errorf("a key retaken %d times was armed again: %v", delivery.MaxRetakes, cc.waits)
+		}
+		if cc := restarted(1); len(cc.waits) != 1 || cc.waits[0] != 20*time.Second {
+			t.Errorf("a key retaken once was armed with %v, want one wait of 20s", cc.waits)
+		}
+	})
+	// AC4: a failure of the source (a compile error, FailedUntilChanged) is never taken again by the backoff.
+	t.Run("a failure of the source is never taken again", func(t *testing.T) {
+		for _, reason := range []string{"go build of tool: undefined: x", delivery.NotPinned + "up: names no digest", delivery.VersionTaken + "@x/y@1 holds another file"} {
+			r, c := retakeRig(t)
+			r.bm.fails["ace/build_go_build"] = reason
+			v := r.consider(t, "modules/m")
+			if v.State != delivery.BuildFailed || delivery.Retakeable(v.Error) {
+				t.Fatalf("%q: build = %+v, want failed and not retakeable", reason, v)
+			}
+			if len(c.waits) != 0 || c.fire() != 0 {
+				t.Errorf("%q: the backoff was armed: %v", reason, c.waits)
+			}
+		}
+	})
+	// Only the backoff's claim counts: a look another event makes while the backoff waits (a toolchain change, a new modules
+	// value) takes the key again and leaves the count as it was, so those looks cannot spend the retakes inside the pause.
+	t.Run("another event's look leaves the count", func(t *testing.T) {
+		r, c := retakeRig(t)
+		r.bm.fails["ace/build_go_build"] = lookupFailure
+		r.consider(t, "modules/m")
+		for i := 0; i < 2*delivery.MaxRetakes; i++ {
+			r.b.Reconsider(context.Background())
+			r.b.Wait()
+		}
+		if v := valueOf(t, r.st, r.key); v.Retakes != 0 || v.State != delivery.BuildFailed {
+			t.Fatalf("after Reconsider looks the key = %+v, want failed with Retakes 0", v)
+		}
+		if c.fire() != 1 {
+			t.Fatal("the backoff's own wait was not armed once")
+		}
+		r.b.Wait()
+		if v := valueOf(t, r.st, r.key); v.Retakes != 1 {
+			t.Errorf("after the backoff's claim the key = %+v, want Retakes 1", v)
+		}
+	})
+	// The journal says each retake with its count and the log's entry of the build's end names it.
+	t.Run("the journal and the log say the count", func(t *testing.T) {
+		r, c := retakeRig(t)
+		var said []string
+		r.b.Logf = func(f string, a ...any) { said = append(said, strings.TrimSpace(fmt.Sprintf(f, a...))) }
+		var ended []delivery.BuildLogEntry
+		r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) {
+			if e.Event == delivery.BuildEndedEvent {
+				ended = append(ended, e)
+			}
+		}
+		r.bm.fails["ace/build_go_build"] = lookupFailure
+		r.consider(t, "modules/m")
+		c.fire()
+		r.b.Wait()
+		if len(ended) != 2 || ended[0].Retakes != 0 || ended[1].Retakes != 1 {
+			t.Fatalf("ended entries = %+v, want retakes 0 then 1; journal: %v", ended, said)
+		}
+		if joined := strings.Join(said, "\n"); !strings.Contains(joined, "taken again after its backoff (retake 1 of 3)") {
+			t.Errorf("the journal says %q, want the retake with its count", joined)
+		}
+	})
+	// The key's error opens with the retake prefix only when every failing lane failed retakeable.
+	t.Run("the key error opens with the prefix only when every lane is retakeable", func(t *testing.T) {
+		for _, c := range []struct {
+			name   string
+			errs   []string
+			prefix bool
+		}{
+			{"one retakeable lane", []string{"mesh-build-go on ace: the step failed: " + lookupFailure}, true},
+			{"two retakeable lanes", []string{"a: " + lookupFailure, "b: " + lookupFailure}, true},
+			{"a retakeable and a compile error", []string{"a: " + lookupFailure, "b: go build of tool: undefined: x"}, false},
+			{"a compile error", []string{"b: go build of tool: undefined: x"}, false},
+		} {
+			got := keyError(c.errs)
+			if has := strings.HasPrefix(got, delivery.Retake); has != c.prefix || delivery.Retakeable(got) != c.prefix {
+				t.Errorf("%s: keyError = %q, want prefix %v", c.name, got, c.prefix)
+			}
+		}
+	})
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -79,6 +79,12 @@ type Builder struct {
 	// time.Now. Node is the node the builder runs on, which asked the build.
 	Now  func() time.Time
 	Node string
+	// RetakeBackoff is the wait before the builder takes a key that failed retakeable (delivery.Retakeable) again by itself,
+	// doubled with each retake (issue 1748): DefaultRetakeBackoff when 0. After runs f once d has passed and returns a
+	// stop; nil waits a context's bound (ADR 0437 §8: the builder holds no timer). The wait is re-armed from the key's value at
+	// start (Sweep): a build's work is not a message of the builder's own on a stream, so there is no delivery delay to ride.
+	RetakeBackoff time.Duration
+	After         func(d time.Duration, f func()) (stop func())
 
 	once     sync.Once
 	sem      chan struct{}
@@ -102,8 +108,14 @@ type Builder struct {
 	waiting map[string]map[string]chan string
 	// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
 	hands map[string]*handWatch
+	// armed are the keys with a retake wait running, by build key; retaking are the keys whose backoff has passed and whose
+	// next claim is the backoff's own (ClaimOptions.Retaking).
+	armed, retaking map[string]bool
 }
 
+// DefaultRetakeBackoff is the first wait before a retakeable failure is taken again; each retake doubles it.
+const DefaultRetakeBackoff = 60 * time.Second
+
 type planned struct {
 	plan Plan
 	err  error
@@ -120,6 +132,7 @@ func (b *Builder) init() {
 		b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
 		b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
 		b.hands = map[string]*handWatch{}
+		b.armed, b.retaking = map[string]bool{}, map[string]bool{}
 	})
 }
 
@@ -187,6 +200,14 @@ func (b *Builder) Sweep(ctx context.Context) error {
 			continue
 		}
 		newly = b.notePublished(cur) || newly
+		if current[key] && cur.State == delivery.BuildFailed {
+			if module, _, err := delivery.ParseBuilderBuildKey(key); err == nil {
+				b.mu.Lock()
+				fp := b.known[module].FP2
+				b.mu.Unlock()
+				b.armRetake(ctx, key, module, fp, cur) // a restarted builder reads the count and goes on
+			}
+		}
 		if current[key] || cur.State != delivery.BuildBuilding {
 			continue
 		}
@@ -429,9 +450,11 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	trunk := b.trunkFP[key]
 	first := !b.tried[key]
 	b.tried[key] = true
+	retaking := b.retaking[key]
+	delete(b.retaking, key)
 	b.mu.Unlock()
 	cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
-		Stored: b.storedOnce(key)})
+		Stored: b.storedOnce(key), Retaking: retaking})
 	if err != nil {
 		b.logf("%s: %v", module, err)
 		return
@@ -477,7 +500,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 		b.logf("%s: left building, the toolchains moved under it", cl.Key)
 		return
 	}
-	result.By, result.Manifest = b.Claimer.By, m.Manifest
+	result.By, result.Manifest, result.Retakes = b.Claimer.By, m.Manifest, cl.Build.Retakes
 	if len(cl.Gone) > 0 && result.State == delivery.BuildDone {
 		result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
 	}
@@ -493,6 +516,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
 	if result.State == delivery.BuildFailed {
 		b.logf("%s: failed: %s", cl.Key, result.Error)
+		b.armRetake(ctx, cl.Key, module, m.FP2, result)
 	}
 	if result.State == delivery.BuildQueued {
 		b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits)
@@ -503,6 +527,57 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	}
 }
 
+// armRetake takes a key that failed retakeable again by itself once its backoff has passed, as a new revision (issue 1748):
+// only an error that opens with delivery.Retake, and at most delivery.MaxRetakes times, counted on the key's value. A key
+// failed for its source (a compile error, FailedUntilChanged) is never armed.
+func (b *Builder) armRetake(ctx context.Context, key, module, fp string, v delivery.BuilderBuild) {
+	if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes >= delivery.MaxRetakes || ctx.Err() != nil {
+		return
+	}
+	b.mu.Lock()
+	if b.armed[key] {
+		b.mu.Unlock()
+		return
+	}
+	b.armed[key] = true
+	b.mu.Unlock()
+	wait := b.RetakeBackoff
+	if wait <= 0 {
+		wait = DefaultRetakeBackoff
+	}
+	wait <<= v.Retakes
+	b.logf("%s: failed retakeable; taken again in %s (retake %d of %d)", key, wait, v.Retakes+1, delivery.MaxRetakes)
+	after := b.After
+	if after == nil {
+		// The builder holds no timer (ADR 0437 §8: a wait is a context's bound): the wait is the bound of a context, which
+		// the builder's own context ends early.
+		after = func(d time.Duration, f func()) func() {
+			wait, stop := context.WithTimeout(ctx, d)
+			go func() {
+				<-wait.Done()
+				if errors.Is(wait.Err(), context.DeadlineExceeded) && ctx.Err() == nil {
+					f()
+				}
+			}()
+			return stop
+		}
+	}
+	after(wait, func() {
+		b.mu.Lock()
+		delete(b.armed, key)
+		m, known := b.known[module]
+		b.mu.Unlock()
+		if ctx.Err() != nil || !known || m.FP2 != fp {
+			return // a newer fingerprint supersedes it: that one is built by its own event
+		}
+		b.logf("%s: taken again after its backoff (retake %d of %d)", key, v.Retakes+1, delivery.MaxRetakes)
+		b.mu.Lock()
+		b.retaking[key] = true
+		b.mu.Unlock()
+		b.consider(ctx, module, m)
+	})
+}
+
 // stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
 // has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
 func (b *Builder) stale(m delivery.Module, plan Plan) string {
@@ -724,6 +799,19 @@ func keyError(errs []string) string {
 			}
 		}
 	}
+	// A retakeable failure opens the key's error only when every failing lane is one (issue 1748): a lane that failed on its
+	// source would be built again for nothing.
+	all := len(errs) > 0
+	for _, e := range errs {
+		all = all && strings.Contains(e, delivery.Retake)
+	}
+	if all {
+		stripped := make([]string, len(errs))
+		for i, e := range errs {
+			stripped[i] = strings.Replace(e, delivery.Retake, "", 1)
+		}
+		return delivery.Retake + strings.Join(stripped, "; ")
+	}
 	return strings.Join(errs, "; ")
 }
 
@@ -955,6 +1043,7 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
 		e.Against = append([]string(nil), m.Inputs2.Bases...)
 	}
 	e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
+	e.Retakes = result.Retakes
 	e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
 	return e
 }

modules/mesh-delivery-builder/manifest_test.go

@@ -212,4 +212,25 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
 	if files < 1 {
 		t.Fatalf("read %d source files, want the program's", files)
 	}
+	// The one bounded wait the builder holds for its own sake is the retake backoff (issue 1748): a goroutine that waits a
+	// context's deadline and then acts. It lives in Builder.armRetake only; any other source holding that idiom is refused.
+	const idiom = "context.DeadlineExceeded"
+	var holders []string
+	for _, root := range []string{"cmd", "internal"} {
+		filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+			if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+				return err
+			}
+			src, _ := os.ReadFile(path)
+			for _, fn := range strings.Split(string(src), "\nfunc ") {
+				if strings.Contains(fn, idiom) && !strings.HasPrefix(fn, "(b *Builder) armRetake(") {
+					holders = append(holders, path)
+				}
+			}
+			return nil
+		})
+	}
+	if len(holders) != 0 {
+		t.Errorf("%v waits for a context's deadline and acts: the retake backoff in Builder.armRetake is the one allowed bounded wait (issue 1748); any other wait is a timer", holders)
+	}
 }