Merge pull request 'Wait out a resolver that does not answer in a build's registry step, and say it was the resolver (issue 1748)' (#447) from fix/1748-registry-wait-retries-a-lookup into main

A build heard and recorded and not registered kept its refusal only when a
plan was open or the plans were held. A tier's first failure fails its plan,
so later refusals were lost; a Retry then settled those builds built from
their records, sent the modules' old versions and reported the plan done.
A build without a readable manifest never reached its plan at all.

Keep the refusal in the store and the process whatever the plans' state, and
fail the asking plan by the build's id when the module is not known.

Issue 1713.

cmd/mesh-controller/builder_says_default_test.go

@@ -0,0 +1,9 @@
+package main
+
+import "context"
+
+// A test's builder says it writes references, as R4e's does, unless the test says otherwise (issue 1886): no test reads
+// the bus for it.
+func init() {
+	builderSaysReferences = func(context.Context) (bool, error) { return true, nil }
+}

cmd/mesh-controller/builder_says_test.go

@@ -0,0 +1,243 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// theBuilderSays sets what the running builder says for one test.
+func theBuilderSays(t *testing.T, says bool, err error) {
+	t.Helper()
+	was := builderSaysReferences
+	builderSaysReferences = func(context.Context) (bool, error) { return says, err }
+	t.Cleanup(func() { builderSaysReferences = was })
+}
+
+// askingBuildAgent records the modules build-agent is asked for, by path.
+func askingBuildAgent(t *testing.T) *[]string {
+	t.Helper()
+	var asked []string
+	was := askABuild
+	askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
+		asked = append(asked, path)
+		return "build-9", nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	return &asked
+}
+
+// 1886/AC2: while no running builder says it writes references, a walk asks build-agent for a module of its tier, as
+// before R4e, and takes nothing from the builder's key.
+func TestAWalkAsksBuildAgentWhileTheBuilderDoesNotSayItWritesReferences(t *testing.T) {
+	theBuilderSays(t, false, nil)
+	asked := askingBuildAgent(t)
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	s := p.Modules["sudo"]
+	if s.Key || s.Build != "build-9" || s.State != "asked" || len(*asked) != 1 {
+		t.Fatalf("the module is not asked of build-agent: %+v, asked %v", s, *asked)
+	}
+}
+
+// 1886/AC3: once a running builder says it writes references, a walk takes the module from the builder's key (ADR
+// 0498 §8) and asks build-agent for nothing.
+func TestAWalkTakesTheKeyOnceTheBuilderSaysItWritesReferences(t *testing.T) {
+	theBuilderSays(t, true, nil)
+	asked := askingBuildAgent(t)
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	if s := p.Modules["sudo"]; !s.Key || s.Build != "" || len(*asked) != 0 {
+		t.Fatalf("the module is not taken from its key: %+v, asked %v", s, *asked)
+	}
+}
+
+// 1886/AC2: a say that cannot be read leaves the module unasked and says why, so the next look asks it: neither path is
+// guessed.
+func TestASayNotReadLeavesTheModuleUnasked(t *testing.T) {
+	theBuilderSays(t, false, errors.New("the bus did not answer"))
+	asked := askingBuildAgent(t)
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	s := p.Modules["sudo"]
+	if s.State != "" || s.Key || len(*asked) != 0 || !strings.Contains(s.Why, "the bus did not answer") {
+		t.Fatalf("a say not read: %+v, asked %v", s, *asked)
+	}
+	if got := unaskedOf(*p); len(got) != 1 {
+		t.Fatalf("the module is not left for the next look to ask: %v", got)
+	}
+}
+
+// 1886/AC2: the one function every ask goes through lets build-agent build any module while the builder does not say
+// it writes references, and refuses it again once it does.
+func TestBuildAgentIsAskedForAnyModuleWhileTheBuilderDoesNotSay(t *testing.T) {
+	was := askedModuleOf
+	t.Cleanup(func() { askedModuleOf = was })
+	askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
+		return strings.TrimPrefix(path, "modules/"), nil
+	}
+	theBuilderSays(t, false, nil)
+	if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo"); err != nil {
+		t.Fatalf("an ask while the builder does not say is refused: %v", err)
+	}
+	theBuilderSays(t, true, nil)
+	if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo"); !errors.Is(err, errNotTheBuildersOwn) {
+		t.Fatalf("an ask once the builder says is not refused: %v", err)
+	}
+}
+
+// 1886/AC4: an open walk that waits on a key the running builder cannot reference (done without a reference, as R4e's
+// own walk did in plan-1792707174627079192) is judged again at the next look: the module is handed back unasked, and
+// that look's ask goes to build-agent. With the builder saying it writes references, the same walk keeps waiting.
+func TestAnOpenWalkWaitingOnAnUnreferencedKeyIsAskedOfBuildAgent(t *testing.T) {
+	t.Cleanup(func() {
+		keyTakes.Lock()
+		keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
+		keyTakes.Unlock()
+	})
+	fp := "src2:" + strings.Repeat("e", 64)
+	unreferenced := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1,
+		Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64),
+			Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: "sha256:1"}}}}
+	walk := func() (*inventory.Plan, map[string]inventory.Entry, planKeysOf) {
+		theBuilderSays(t, true, nil)
+		p, byName := aWalkOfOne("sudo")
+		askModule(t.Context(), p, "sudo", byName)
+		keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
+			keys: map[string]builderkeys.Key{"sudo": unreferenced}}
+		return p, byName, keys
+	}
+	take := func(string, sdkdelivery.Module, string) { t.Fatal("an unreferenced key was taken in") }
+	records := recordsOf{builds: map[string]inventory.Build{}}
+
+	// Said: it waits for the reference, as ADR 0498 §8 has it.
+	p, _, keys := walk()
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
+		t.Fatal(err)
+	}
+	if s := p.Modules["sudo"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "waits for the reference") {
+		t.Fatalf("with the builder saying, the walk does not wait for the reference: %+v", s)
+	}
+
+	// Not said: handed back, and the next look asks build-agent.
+	p, byName, keys := walk()
+	theBuilderSays(t, false, nil)
+	asked := askingBuildAgent(t)
+	changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take)
+	if err != nil || !changed {
+		t.Fatalf("the walk was not judged again: %v %v", changed, err)
+	}
+	s := p.Modules["sudo"]
+	if s.Key || s.State != "" || p.State != inventory.PlanBuilding || !strings.Contains(s.Why, "does not say") {
+		t.Fatalf("the module is not handed back to be asked of build-agent: %+v %s", s, p.State)
+	}
+	for _, m := range unaskedOf(*p) {
+		askModule(t.Context(), p, m, byName)
+	}
+	if s := p.Modules["sudo"]; s.Key || s.Build != "build-9" || len(*asked) != 1 {
+		t.Fatalf("the next look did not ask build-agent: %+v, asked %v", s, *asked)
+	}
+}
+
+// 1886/AC4: a key the running builder failed while it does not say it writes references fails no walk: the module is
+// asked of build-agent, whose build is the one delivered.
+func TestAFailedKeyOfABuilderThatDoesNotSayFailsNoWalk(t *testing.T) {
+	theBuilderSays(t, true, nil)
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	_ = byName
+	fp := "src2:" + strings.Repeat("f", 64)
+	theBuilderSays(t, false, nil)
+	keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
+		keys: map[string]builderkeys.Key{"sudo": {Module: "sudo", Fingerprint: fp, Found: true,
+			Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: no reference"}}}}
+	if _, err := settleKeys(t.Context(), recordsOf{}, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
+		t.Fatal(err)
+	}
+	if s := p.Modules["sudo"]; p.State != inventory.PlanBuilding || s.State != "" || s.Key {
+		t.Fatalf("a failed key of a builder that does not say failed the walk: %s %q %+v", p.State, p.Note, s)
+	}
+}
+
+// 1886/AC6 (ADR 0522 §6): the parts that write references, the builder and the build modules, are build-agent's own until
+// R4g: asked of build-agent and never taken from keys they write themselves, whatever the builder says.
+func TestTheReferenceWritersAreAskedOfBuildAgentWhateverTheBuilderSays(t *testing.T) {
+	theBuilderSays(t, true, nil)
+	asked := askingBuildAgent(t)
+	for _, m := range []string{"mesh-build-go", "mesh-build-node", "mesh-build-image", "mesh-delivery-builder"} {
+		p, byName := aWalkOfOne(m)
+		askModule(t.Context(), p, m, byName)
+		if s := p.Modules[m]; s.Key || s.Build != "build-9" {
+			t.Errorf("%s is not asked of build-agent: %+v", m, s)
+		}
+	}
+	if len(*asked) != 4 {
+		t.Fatalf("build-agent was asked %v", *asked)
+	}
+}
+
+// 1886 review round 1 (ADR 0522 §2): the guard of the one ask function judges by the say askModule read, never a second
+// read: a second read that fails, or that says otherwise, fails no plan.
+func TestTheAskGuardUsesTheSayTheWalkRead(t *testing.T) {
+	was := askedModuleOf
+	t.Cleanup(func() { askedModuleOf = was })
+	askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
+		return strings.TrimPrefix(path, "modules/"), nil
+	}
+	reads := 0
+	wasSays := builderSaysReferences
+	t.Cleanup(func() { builderSaysReferences = wasSays })
+	for _, second := range []struct {
+		name string
+		says bool
+		err  error
+	}{{"a second read that fails", false, errors.New("the bus did not answer")}, {"a say that turned true", true, nil}} {
+		reads = 0
+		builderSaysReferences = func(context.Context) (bool, error) {
+			reads++
+			if reads == 1 {
+				return false, nil
+			}
+			return second.says, second.err
+		}
+		wasAsk := askABuild
+		askABuild = func(ctx context.Context, source buildSource, path, _ string) (string, error) {
+			if err := onlyTheBuildersOwn(ctx, source, path); err != nil {
+				return "", err
+			}
+			return "build-9", nil
+		}
+		p, byName := aWalkOfOne("sudo")
+		askModule(t.Context(), p, "sudo", byName)
+		askABuild = wasAsk
+		if s := p.Modules["sudo"]; p.State == inventory.PlanFailed || s.Build != "build-9" || s.Key {
+			t.Errorf("%s: the walk %s, the module %+v", second.name, p.State, s)
+		}
+	}
+}
+
+// 1886 merge check hang: the look must judge an ask and a settle by one say. A settle that read the say from another
+// source than the ask (the bus's bucket, against the walk's read) handed a module back, the next ask took it from its key
+// again, and the look's loop never ended: walk_from_keys_integration_test hung.
+func TestTheSettleJudgesByTheSameSayAsTheAsk(t *testing.T) {
+	theBuilderSays(t, true, nil)
+	p, _ := aWalkOfOne("sudo")
+	p.Modules["sudo"] = &inventory.PlanModule{}
+	askModule(t.Context(), p, "sudo", map[string]inventory.Entry{"sudo": {Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/sudo", Ref: "main"}}})
+	fp := "src2:" + strings.Repeat("1", 64)
+	keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
+		keys: map[string]builderkeys.Key{"sudo": {Module: "sudo", Fingerprint: fp, Found: true,
+			Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "shanks"}}}}
+	if _, err := settleKeys(t.Context(), recordsOf{}, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
+		t.Fatal(err)
+	}
+	if s := p.Modules["sudo"]; !s.Key || s.State != "asked" {
+		t.Fatalf("the settle judged by another say than the ask's: %+v", s)
+	}
+}

cmd/mesh-controller/main.go

@@ -448,9 +448,12 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
 		return nil
 	case err != nil:
 		fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
-		if manifest.Module != "" {
+		takeRefused(ctx, b.inv, result, manifest.Module, err.Error(), refusedOutcome{
-			planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
+			built: func(module, why string) {
-		}
+				planBuilt(ctx, b.open, module, result.Commit, why, asked, result.ID)
+			},
+			failedBuild: func(failed link.BuildResult) { planFailedBuild(ctx, b.open, failed) },
+		})
 		return nil
 	}
 	fmt.Printf("%s: %s %s registered, built on %s from %s\n",

cmd/mesh-controller/new_modules.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
@@ -209,7 +210,12 @@ func watchNewModules(ctx context.Context, open *stores) {
 		fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
 		return
 	}
-	if err := builderkeys.Watch(ctx, keys.JS, func(module string) { wakeNewModule(ctx, open, module) }); err != nil {
+	says := watchingSays{js: keys.JS}
+	says.watch(ctx)
+	if err := builderkeys.Watch(ctx, keys.JS, func(module string) {
+		says.watch(ctx)
+		wakeNewModule(ctx, open, module)
+	}); err != nil {
 		closeKeys()
 		fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
 		return
@@ -272,3 +278,32 @@ func newModulesOf(ctx context.Context, repository string) ([]string, error) {
 	sort.Strings(out)
 	return out, nil
 }
+
+// watchingSays watches the builder's bucket of instances once it is there (issue 1886): a change of what the builder says
+// asks the look, which judges every open walk again. A builder from before R4e makes no such bucket; a write of the
+// builder's keys asks again whether it is there now, so nothing waits on a timer.
+type watchingSays struct {
+	js jetstream.JetStream
+	mu *sync.Mutex
+	on *bool
+}
+
+func (w *watchingSays) watch(ctx context.Context) {
+	if w.mu == nil {
+		w.mu, w.on = &sync.Mutex{}, new(bool)
+	}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if *w.on {
+		return
+	}
+	on, err := builderkeys.WatchSays(ctx, w.js, askToLook)
+	if err != nil {
+		fmt.Printf("plans: what the builder says it writes is not watched now: %v\n", err)
+		return
+	}
+	if on {
+		*w.on = true
+		askToLook() // what it says now may differ from what the last look read
+	}
+}

cmd/mesh-controller/plan_keys.go

@@ -32,7 +32,12 @@ func isBuildersOwn(module string) bool { return buildersOwn(module) }
 // repository's; and mesh-tools, the toolchain image build-agent's builds stand on. The builder writes no key of any of them,
 // and a walk waiting for one would hold its tier for ever. mesh-tools-go is not among them: it builds from a Dockerfile in
 // its own directory on a pinned base, and has its key from mesh-build-image.
-var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true}
+//
+// **And the parts that write references** (ADR 0522 §6, issue 1886): the build modules push and answer each reference,
+// and the builder writes it into the key, so none of them is ever taken from a key they write themselves. Until R4g
+// build-agent is the way they are delivered whatever the builder says.
+var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true,
+	"mesh-build-go": true, "mesh-build-node": true, "mesh-build-image": true}
 
 // buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may
 // name modules of its own as build-agent's.
@@ -44,6 +49,32 @@ type planKeyReads interface {
 	Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
 }
 
+// builderSayKey carries, on an ask's context, the builder's say the walk read before asking (withBuilderSay).
+type builderSayKey struct{}
+
+// withBuilderSay hands the say a walk read to the ask's guard, so the guard never reads it a second time.
+func withBuilderSay(ctx context.Context, says bool) context.Context {
+	return context.WithValue(ctx, builderSayKey{}, says)
+}
+
+// builderSayOf is the say a walk handed the ask, and whether it handed one.
+func builderSayOf(ctx context.Context) (says, handed bool) {
+	says, handed = ctx.Value(builderSayKey{}).(bool)
+	return says, handed
+}
+
+// builderSaysReferences is whether a running builder instance says, in its own bucket of instances, that the keys it
+// writes name references (issue 1886). Until one does, a walk asks build-agent for every module, as before R4e: the
+// keys the running builder writes could never be delivered from. A variable so a test needs no bus.
+var builderSaysReferences = func(ctx context.Context) (bool, error) {
+	keys, closeKeys, err := openBuilderKeys()
+	if err != nil {
+		return false, err
+	}
+	defer closeKeys()
+	return keys.SaysReferences(ctx)
+}
+
 // openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
 var openPlanKeys = func() (planKeyReads, func(), error) {
 	r, closeKeys, err := openBuilderKeys()
@@ -102,6 +133,20 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
 			changed = true
 		}
 	}
+	// One say for the ask and the settle (builderSaysReferences): two sources that differ hand a module back and take it
+	// from its key again at every turn of the look, which then never ends.
+	says, err := builderSaysReferences(ctx)
+	if err != nil {
+		return changed, fmt.Errorf("whether the builder writes references cannot be read: %w", err)
+	}
+	// **A walk never waits on a key the running builder cannot reference** (issue 1886): while no builder says it writes
+	// references, a module that waits on its key, or whose key failed, is handed back unasked, and the look asks it of
+	// build-agent. A done key with every reference is still taken.
+	toAgent := func(s *inventory.PlanModule, why string) {
+		s.State, s.Key, s.AskedAt, s.KeyAt, s.Build, s.Fingerprint = "", false, nil, "", "", ""
+		s.Why = "the builder does not say it writes references, so build-agent is asked (" + why + ")"
+		changed = true
+	}
 	for _, m := range tier {
 		s := p.Modules[m]
 		if s == nil || !s.Key || s.State != "asked" {
@@ -109,6 +154,10 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
 		}
 		trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
 		if errors.Is(err, builderkeys.ErrNoValue) {
+			if !says {
+				toAgent(s, "the indexer holds no value of it")
+				continue
+			}
 			set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
 			continue
 		}
@@ -120,6 +169,10 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
 			if why == "" {
 				why = "the indexer wrote no fingerprint of the new form"
 			}
+			if !says {
+				toAgent(s, why)
+				continue
+			}
 			set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
 			continue
 		}
@@ -131,6 +184,9 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven
 			return changed, err
 		}
 		switch {
+		case !says && (k.Failed() || !k.Done() || len(k.Unreferenced()) > 0):
+			toAgent(s, k.Says())
+			continue
 		case k.Failed():
 			// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
 			// `version taken:`), and a rebuild of it is a new revision the next walk takes.

cmd/mesh-controller/plan_retry.go

@@ -203,6 +203,30 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
 		return oneWalkAtATime(p, plans, onPath)
 	}
 	stopped := stoppedRollouts(p)
+	// **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is
+	// asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send.
+	if len(stopped) == 0 && stoppedByHandWaiting(p) {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		for _, m := range holdingABuild(p) {
+			if q, found := newerPlanFor(m, p, plans); found {
+				return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+
+					"over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
+			}
+		}
+		return oneWalkAtATime(p, plans, onPath)
+	}
+	// **A walk lost to silence is resumed** (novox/hq ADR 0531, issue 1886): its tier was sent whole and only the gate after it failed it, every
+	// machine it waited on silent; once they reported, it goes on from that gate.
+	if len(stopped) == 0 && lostToSilence(p) {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would send "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		return oneWalkAtATime(p, plans, onPath)
+	}
 	if len(stopped) == 0 {
 		return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
 			p.Tier, p.ID, p.Note)
@@ -333,6 +357,8 @@ func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inv
 
 // retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
 // that tier: what follows is the plan going on as if they had built the first time.
+// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again
+// (resumeStopped, issue 1886).
 func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	inv := open.inventory
 	release, err := inv.HoldPlans(ctx, true)
@@ -388,6 +414,27 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
 		return retryTierWhole(ctx, open, &p, again)
 	}
+	if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) {
+		said := resumeStopped(&p)
+		if err := inv.SavePlan(ctx, &p); err != nil {
+			return "", err
+		}
+		return said, nil
+	}
+	if len(failed) == 0 && lostToSilence(p) {
+		reports, err := inv.LastReports(ctx)
+		if err != nil {
+			return "", err
+		}
+		said, err := resumeLost(&p, silentNodes(p, reports, time.Now().UTC(), provLookup(ctx)))
+		if err != nil {
+			return "", err
+		}
+		if err := inv.SavePlan(ctx, &p); err != nil {
+			return "", err
+		}
+		return said, nil
+	}
 	if len(failed) == 0 {
 		return retryRollouts(ctx, open, &p)
 	}
@@ -583,3 +630,137 @@ func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[
 	}
 	return failedIn(*p)
 }
+
+// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a
+// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left
+// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299).
+func stoppedByHandWaiting(p inventory.Plan) bool {
+	if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" ||
+		p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 {
+		return false
+	}
+	asked, unsent, unasked := waitingIn(p)
+	return asked+unsent+unasked > 0
+}
+
+// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked.
+func waitingIn(p inventory.Plan) (asked, unsent, unasked int) {
+	if p.Tier >= len(p.Tiers) {
+		return 0, 0, 0
+	}
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		switch {
+		case s == nil || s.State == "":
+			unasked++
+		case s.State == "asked":
+			asked++
+		case s.State == "built" && s.SentAt == nil:
+			unsent++
+		}
+	}
+	return asked, unsent, unasked
+}
+
+// holdingABuild is the modules of a stopped walk's current tier that hold a build of its commit, not yet sent: built and
+// not sent, or asked under a build id. Only these could put an older build over a newer plan's (review of issue 1886): a
+// module not yet asked is asked at its followed branch's head when the walk reaches it, and one asked on the builder's key
+// is settled by the look from the key of the commit it reads then. Sorted.
+func holdingABuild(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		if s == nil || s.SentAt != nil || s.Key {
+			continue
+		}
+		if s.State == "built" || (s.State == "asked" && s.Build != "") {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept but
+// build-agent's own (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the
+// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it.
+func resumeStopped(p *inventory.Plan) string {
+	// **Build-agent's own are asked of build-agent** (novox/hq ADR 0522 §6): one marked taken from a key it writes itself is
+	// set unasked, and the look asks the rest of a partly asked tier (unaskedOf), of build-agent, whatever the builder says.
+	var again []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "asked" && s.Key && isBuildersOwn(m) {
+			s.State, s.Key, s.AskedAt, s.Build, s.Why = "", false, nil, "", ""
+			again = append(again, m)
+		}
+	}
+	sort.Strings(again)
+	asked, unsent, unasked := waitingIn(*p)
+	later := 0
+	for i := p.Tier + 1; i < len(p.Tiers); i++ {
+		later += len(p.Tiers[i])
+	}
+	by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy
+	waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after",
+		asked, unsent, unasked, later)
+	if len(again) > 0 {
+		waits += "; " + strings.Join(again, ", ") + " set unasked, for the look to ask of build-agent (its own, ADR 0522 §6)"
+	}
+	p.State = inventory.PlanBuilding
+	goesOnAgain(p)
+	p.Delivery.Kept = nil
+	p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits)
+	return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+
+		"on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits)
+}
+
+// silentGateWords say, in a plan's note, that the gate between two tiers found every machine it waits on silent past
+// ApplySilentAfter, and failed the plan as lost (issue 1160, ADR 0531): one text for the writer and the reader (lostToSilence).
+const silentGateWords = "every machine it waits on said nothing on its send within"
+
+// lostToSilence is whether a plan failed only at the gate after its tier, every machine it waited on silent (ADR 0531,
+// issue 1886): nothing in its tier failed to build, stopped rolling out or is still to ask — the look had sent the whole tier.
+func lostToSilence(p inventory.Plan) bool {
+	if p.State != inventory.PlanFailed || p.Tier >= len(p.Tiers) || !strings.Contains(p.Note, silentGateWords) ||
+		len(failedIn(p)) > 0 || len(stoppedRollouts(p)) > 0 {
+		return false
+	}
+	asked, _, unasked := waitingIn(p)
+	return asked == 0 && unasked == 0
+}
+
+// resumeLost sets a walk lost to silence building again at the tier it failed after (ADR 0531, issue 1886), for the next look to
+// judge its gate again on the machines' reports, unless every machine its tier was sent to is still silent: then
+// nothing it needs is delivered anywhere yet, and the retry is refused as the gate would fail it again.
+func resumeLost(p *inventory.Plan, silent map[string]bool) (string, error) {
+	var sent, still []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil {
+			for n := range s.Rest {
+				if !slices.Contains(sent, n) {
+					sent = append(sent, n)
+				}
+			}
+		}
+	}
+	for _, n := range sent {
+		if silent[n] {
+			still = append(still, n)
+		}
+	}
+	sort.Strings(still)
+	if len(sent) > 0 && len(still) == len(sent) {
+		return "", fmt.Errorf("%s failed because every machine its tier %d waits on said nothing on its send, and %s "+
+			"still said nothing: retry once it reported", p.ID, p.Tier+1, strings.Join(still, ", "))
+	}
+	was := p.Note
+	p.State = inventory.PlanBuilding
+	goesOnAgain(p)
+	p.Note = fmt.Sprintf("tier %d resumed by hand after it failed as lost (%s): its gate is judged again on the "+
+		"machines' reports", p.Tier, was)
+	return fmt.Sprintf("%s resumed at tier %d of %d, failed as lost to silence (%s); this verb asks and sends nothing — "+
+		"the next look judges its gate again and carries it on as any walk", p.ID, p.Tier+1, len(p.Tiers), was), nil
+}

cmd/mesh-controller/plan_retry_stopped_test.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next
+// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds
+// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do
+// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on.
+func TestRetryResumesAWalkStoppedByHand(t *testing.T) {
+	at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
+	sent := at.Add(-time.Hour)
+	p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
+		Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
+		Note: "stopped by a person at tier 1: issue 1886",
+		Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886",
+			Kept: []inventory.PlanKept{{Module: "mesh-controller"}}},
+		Modules: map[string]*inventory.PlanModule{}}
+	tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"}
+	for _, m := range tier0 {
+		p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+	}
+	var tier1 []string
+	for _, m := range []string{"build-agent", "mesh-tools"} {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "built", AskedAt: &sent, BuiltAt: &sent, Build: "build-" + m}
+	}
+	onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker",
+		"mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"}
+	for _, m := range onKeys {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m}
+	}
+	tier2 := []string{"gitea", "mesh-catalog", "mongodb"}
+	for _, m := range tier2 {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	p.Tiers = [][]string{tier0, tier1, tier2}
+
+	if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
+		t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err)
+	}
+	said := resumeStopped(&p)
+	if p.State != inventory.PlanBuilding {
+		t.Errorf("resumed walk is %s, wanted building", p.State)
+	}
+	if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 {
+		t.Errorf("resumed walk still says it stopped: %+v", p.Delivery)
+	}
+	if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") {
+		t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note)
+	}
+	for _, m := range onKeys {
+		s := p.Modules[m]
+		if isBuildersOwn(m) {
+			// Build-agent's own are asked of build-agent whatever the builder says (ADR 0522 §6): set unasked, for the look.
+			if s.State != "" || s.Key || s.Build != "" || s.AskedAt != nil {
+				t.Errorf("%s, build-agent's own, was left on the builder's key: %+v", m, s)
+			}
+			if !slices.Contains(unaskedOf(p), m) {
+				t.Errorf("%s is not among what the look asks next: %v", m, unaskedOf(p))
+			}
+			continue
+		}
+		if s.State != "asked" || !s.Key || s.Build != "build-"+m {
+			t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s)
+		}
+	}
+	for _, w := range []string{"plan-r4e", "tier 2 of 3", "5 asked", "2 built and not sent", "3 not yet asked in it",
+		"3 not yet asked in the tiers after", "mesh-build-go, mesh-build-image, mesh-build-node set unasked"} {
+		if !strings.Contains(said, w) {
+			t.Errorf("the answer %q does not say %q", said, w)
+		}
+	}
+
+	// What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside.
+	done := p
+	done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	done.Modules = map[string]*inventory.PlanModule{}
+	for _, tier := range done.Tiers {
+		for _, m := range tier {
+			done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+		}
+	}
+	if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") {
+		t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err)
+	}
+	stopped := p
+	stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
+		Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
+		Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("a stopped walk was resumed beside a related open walk: %v", err)
+	}
+	// A newer open plan of its repository and branch supersedes it: resuming would build what that one replaced.
+	newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
+		Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
+		Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{stopped, newerOpen}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "plan-newer supersedes it") {
+		t.Errorf("a stopped walk was resumed under a newer open plan of its branch: %v", err)
+	}
+	// A newer plan holding a build of its tier not yet sent refuses it: resuming would send the older build over the newer.
+	newerBuilt := inventory.Plan{ID: "plan-agent", Repository: "novox/mesh-controller", Branch: "main", Commit: "d00dfeed",
+		Created: at, State: inventory.PlanDone, Tiers: [][]string{{"build-agent"}},
+		Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{stopped, newerBuilt}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "older build") {
+		t.Errorf("a stopped walk was resumed over a newer plan of a build it has not sent: %v", err)
+	}
+}
+
+// A newer plan holding a module the stopped walk has not yet asked does not refuse its resume (review of issue 1886): that
+// module is asked at its followed branch's head when the walk reaches it, so no older build goes over the newer. Open and
+// related, the newer walk refuses it under one walk at a time; done, it does not. One holding a build the walk has not sent
+// still refuses it.
+func TestResumeIsNotRefusedByANewerPlanOfAModuleNotYetAsked(t *testing.T) {
+	at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
+	sent := at.Add(-time.Hour)
+	p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
+		Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
+		Delivery: &inventory.PlanDelivery{Stopped: "a person", StoppedWhy: "issue 1886"},
+		Tiers:    [][]string{{"mesh-sdk"}, {"build-agent", "mesh-delivery-gate"}, {"mesh-delivery-builder", "gitea"}},
+		Modules: map[string]*inventory.PlanModule{
+			"mesh-sdk":              {State: "built", BuiltAt: &sent, SentAt: &sent},
+			"build-agent":           {State: "built", BuiltAt: &sent},
+			"mesh-delivery-gate":    {State: "asked", AskedAt: &sent, Key: true},
+			"mesh-delivery-builder": {}, "gitea": {}}}
+	newer := func(state string, holds string) inventory.Plan {
+		return inventory.Plan{ID: "plan-1792715180015327197", Repository: "novox/mesh-delivery", Branch: "main",
+			Commit: "d00dfeed", Created: at, State: state, Tiers: [][]string{{holds}},
+			Modules: map[string]*inventory.PlanModule{holds: {State: "asked"}}}
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-builder")}, ownPathOf(nil)); err != nil {
+		t.Errorf("a done newer plan of a module not yet asked refused the resume: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-gate")}, ownPathOf(nil)); err != nil {
+		t.Errorf("a done newer plan of a module on the builder's key refused the resume: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanBuilding, "mesh-delivery-builder")}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("an open newer walk related to it did not refuse the resume under one walk at a time: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "build-agent")}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "older build") {
+		t.Errorf("a newer plan of a built module not yet sent did not refuse the resume: %v", err)
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -260,7 +260,7 @@ type gateNeed struct {
 }
 
 // silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent
-// them (novox/hq ADR 0360 §5, issue 1160): the one rule both the gates between tiers and the walk's end read, through
+// them (novox/hq ADR 0360 §7, issue 1160): the one rule both the gates between tiers and the walk's end read, through
 // walkReports. A machine that reports on its send is not silent from then on.
 func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool {
 	_, _, silent := walkReports(p, reports, now, prov)
@@ -272,7 +272,7 @@ func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time,
 }
 
 // withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard
-// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §5).
+// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §7).
 func withoutSilent(running []string, silent map[string]bool) []string {
 	return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] })
 }
@@ -457,6 +457,18 @@ func onlyTheBuildersOwn(ctx context.Context, source buildSource, path string) er
 	if isBuildersOwn(module) {
 		return nil
 	}
+	// **While no running builder says it writes references, build-agent builds every module** (issue 1886): judged by the
+	// say the walk read when it has one, read here only for an ask that carries none.
+	says, read := builderSayOf(ctx)
+	if !read {
+		var err error
+		if says, err = builderSaysReferences(ctx); err != nil {
+			says = true // not known: refused below, as before the say existed
+		}
+	}
+	if !says {
+		return nil
+	}
 	if module == "" {
 		module = fmt.Sprintf("%s at %q", source, orRoot(path))
 	}
@@ -529,11 +541,23 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
 	}
 	// **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503
 	// §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys).
+	// **Only while a running builder says it writes references** (issue 1886): until then the keys it writes could never
+	// be delivered from, and the module is asked of build-agent as before R4e. A say not read leaves it unasked, said,
+	// for the next look.
 	if !isBuildersOwn(name) {
+		says, err := builderSaysReferences(ctx)
+		if err != nil {
+			state.Why = "waits: whether the builder writes references cannot be read: " + firstLine(err.Error())
+			return
+		}
+		if says {
 			sayBuilding(ctx, p.ID, now)
 			takeFromKey(p, state, e, now)
 			return
 		}
+		// The ask's guard judges by this say, never by a second read that could fail or differ (review of issue 1886).
+		ctx = withBuilderSay(ctx, says)
+	}
 	source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
 	fmt.Printf("  tier %d: ", p.Tier)
 	// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every
@@ -1105,7 +1129,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// being behind. Now every module of the tier whose first machine is the same is sent there in one
 	// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
 	// of the tier's machines are sent once, together, when the gates have passed.
-	// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §5, issue 1160): the
+	// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §7, issue 1160): the
 	// walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it.
 	allReports, err := inv.LastReports(ctx)
 	if err != nil {
@@ -1433,8 +1457,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			// Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does
 			// not go on as if it had been (issue 1160).
 			p.State = inventory.PlanFailed
-			p.Note = fmt.Sprintf("tier %d built; every machine it waits on said nothing on its send within %s: %s",
+			p.Note = fmt.Sprintf("tier %d built; %s %s: %s", p.Tier, silentGateWords, applySilentAfter, strings.Join(quiet, ", "))
-				p.Tier, applySilentAfter, strings.Join(quiet, ", "))
 			fmt.Printf("%s: %s\n", p.ID, p.Note)
 			return true, nil
 		}
@@ -1452,7 +1475,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			return changed, nil
 		}
 	}
-	// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent
+	// **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §7): after its last tier, every machine it sent
 	// has reported on that send, or is silent past ApplySilentAfter; never a health reading.
 	end := ""
 	if p.Tier == len(p.Tiers)-1 {
@@ -1519,26 +1542,35 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
 			carried[node] = append(carried[node], m)
 		}
 	}
-	seen := map[string]bool{}
+	// **Each machine on the walk's last send to it** (novox/hq ADR 0531, issue 1886): a send carries the machine's whole declaration (ADR
+	// 0221), so a report on the last one answers every earlier one, and the bound runs from it. Judged on every send
+	// it had made, a walk whose tiers were sent hours apart — R4e, stopped at tier 1 and resumed — held tier 0's send of
+	// 00:23 against tier 1's templates and named its machines silent 35 seconds after tier 1 was sent.
+	last := map[string]inventory.SentDeclaration{}
+	lastAt := map[string]time.Time{}
 	for _, s := range p.Modules {
 		if s == nil {
 			continue
 		}
 		for node, sent := range s.Rest {
-			sent = sent.Only(carried[node]...)
-			if seen[node] {
-				continue
-			}
-			r := byNode[node]
 			at := sent.At
 			if at.IsZero() && s.SentAt != nil {
 				at = *s.SentAt
 			}
+			was, ok := last[node]
+			if !ok || laterSend(sent, at, was, lastAt[node]) {
+				last[node], lastAt[node] = sent, at
+			}
+		}
+	}
+	for node, sent := range last {
+		at := lastAt[node]
+		sent = sent.Only(carried[node]...)
+		r := byNode[node]
 		// A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5).
 		outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov)
 		switch {
 		case reported:
-				seen[node] = true
 			switch {
 			case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused:
 				failed = append(failed, node)
@@ -1546,23 +1578,26 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
 				failed = append(failed, node+" ("+outcome+")")
 			}
 		case !at.IsZero() && now.Sub(at) > applySilentAfter:
-				seen[node] = true
 			silent = append(silent, node)
 		default:
-				if !slices.Contains(waiting, node) {
 			waiting = append(waiting, node)
 		}
 	}
-		}
-	}
-	// A machine one module's send was reported on and another's not yet is the same send: reported.
-	waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] })
 	sort.Strings(waiting)
 	sort.Strings(failed)
 	sort.Strings(silent)
 	return waiting, failed, silent
 }
 
+// laterSend is whether a, sent at aAt, is a later send to a machine than b, sent at bAt: by its sequence when both carry
+// one, else by when it went.
+func laterSend(a inventory.SentDeclaration, aAt time.Time, b inventory.SentDeclaration, bAt time.Time) bool {
+	if a.Sequence > 0 && b.Sequence > 0 && a.Sequence != b.Sequence {
+		return a.Sequence > b.Sequence
+	}
+	return aAt.After(bAt)
+}
+
 // restOf is, for one module, every machine of its rest that the send reached and what it carried there.
 func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration {
 	out := map[string]inventory.SentDeclaration{}

cmd/mesh-controller/replay867_test.go

@@ -44,6 +44,11 @@ func replay867(t *testing.T, recheck bool) {
 	if _, err := exec.LookPath("go"); err != nil {
 		t.Skip("no Go toolchain on this machine to build a judge with")
 	}
+	// The check's floor of free space (novox/hq issue 1834) reads the disk the test runs on, which is none of this
+	// replay's business: a build node under the floor failed every mesh-controller check here (issue 1834, note 12).
+	wasFree := builder.FreeBytes
+	builder.FreeBytes = func(string) (uint64, error) { return 500 << 30, nil }
+	t.Cleanup(func() { builder.FreeBytes = wasFree })
 	r := theGroupOfIssue867(t)
 	for _, c := range []struct {
 		what, repo string

cmd/mesh-controller/take_refused.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// refusalKeeper keeps a refused registration in the store (*inventory.Inventory, or a test's).
+type refusalKeeper interface {
+	RecordRefused(ctx context.Context, id, module, why string) error
+}
+
+// refusedOutcome is how a refused build reaches the plans: built takes it in by its module (planBuilt), failedBuild
+// by the id the plan asked it under (planFailedBuild).
+type refusedOutcome struct {
+	built       func(module, why string)
+	failedBuild func(result link.BuildResult)
+}
+
+// takeRefused takes in a build heard and recorded, and not registered (novox/hq issue 1713).
+//
+// **The refusal is kept by the build's id, whatever the plans' state**: in the store (RecordRefused) and in this
+// process (keepRefusalFor). Taking it into the plans keeps it only when a plan is open or the plans are held, and a
+// tier's first failure fails its plan, which no outcome reaches after that: the build record says the builder's words,
+// which worked, and a Retry settled from it sent the module's old version and reported the plan done (on 2026-10-23
+// 12:01, plan-1792748906937927069 sent mesh-build-go, mesh-build-node, mosquitto and ssh-client so).
+//
+// **A build whose module is not known fails the plan that asked it by its id** (planFailedBuild, moduleAskedAs), as
+// a failed build that names no module does: a manifest that could not be read is no reason for the plan not to hear.
+func takeRefused(ctx context.Context, keeper refusalKeeper, result link.BuildResult, module, why string,
+	plans refusedOutcome) {
+	if result.ID != "" {
+		if err := keeper.RecordRefused(ctx, result.ID, module, why); err != nil {
+			fmt.Printf("%s: the refusal could not be kept in the store, only in this controller: %v\n", result.ID, err)
+		}
+		keepRefusalFor(result.ID, why, time.Now())
+	}
+	if module != "" {
+		plans.built(module, why)
+		return
+	}
+	failed := result
+	failed.Failed = why
+	plans.failedBuild(failed)
+}

cmd/mesh-controller/take_refused_test.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// keptRefusals is a store that keeps refusals, by id.
+type keptRefusals map[string]string
+
+func (k keptRefusals) RecordRefused(_ context.Context, id, _, why string) error {
+	if _, ok := k[id]; !ok {
+		k[id] = why
+	}
+	return nil
+}
+
+// failedPlanOf is a plan already failed on gitea, its tier also asking module under id: open() does not count it, so
+// no outcome taken in reaches it (as plan-1792748906937927069 on 2026-10-23 11:51).
+func failedPlanOf(module, id string, asked time.Time) inventory.Plan {
+	return inventory.Plan{ID: "plan-1713", State: inventory.PlanFailed, Tier: 0,
+		Tiers: [][]string{{"gitea", module}}, Note: "gitea failed to build in tier 0",
+		Modules: map[string]*inventory.PlanModule{
+			"gitea": {State: "failed", AskedAt: &asked, Build: "build-1713-gitea", Why: "not registered"},
+			module:  {State: "asked", AskedAt: &asked, Build: id},
+		}}
+}
+
+// retried is what a Retry of the plan asks again, its records read as recordsOfAsked reads them: the build record of
+// id (which says the builder's words, worked) with every refusal kept for it in the store and in this process.
+func retried(p *inventory.Plan, store keptRefusals, id, module string, at time.Time) []string {
+	byID := map[string]inventory.Build{id: {ID: id, Module: module, Commit: "6c0024e4", At: at}}
+	refused := map[string]string{}
+	for k, why := range store {
+		refused[k] = why
+	}
+	for k, why := range refusalsKeptFor([]string{id}) {
+		refused[k] = why
+	}
+	overlayRefusals(nil, byID, refused)
+	return toRetry(p, nil, byID)
+}
+
+// 1713/AC1: given a build of a module heard and recorded and not registered while its plan is failed and the plans are
+// not held, when the controller takes it in, then the refusal is kept by id, and a Retry asks the module again instead
+// of settling it built and sending its old version (plan-1792748906937927069 sent mosquitto's on 2026-10-23 12:01).
+func TestARefusalWhileThePlanIsFailedIsKeptAndRetried(t *testing.T) {
+	asked := time.Date(2026, 10, 23, 11, 51, 0, 0, time.UTC)
+	id := "build-1713-ac1-mosquitto"
+	p := failedPlanOf("mosquitto", id, asked)
+	store := keptRefusals{}
+	var took []string
+	takeRefused(t.Context(), store, link.BuildResult{ID: id, Commit: "6c0024e4"}, "mosquitto",
+		"the trunk requires no status", refusedOutcome{
+			built:       func(module, why string) { took = append(took, module+": "+why) },
+			failedBuild: func(link.BuildResult) { t.Fatal("a refusal with its module was taken by id") },
+		})
+
+	if !reflect.DeepEqual(took, []string{"mosquitto: the trunk requires no status"}) {
+		t.Fatalf("the plans were told %v", took)
+	}
+	if store[id] == "" {
+		t.Fatal("the refusal was not kept in the store")
+	}
+	if got := retried(&p, keptRefusals{}, id, "mosquitto", asked.Add(time.Minute)); !reflect.DeepEqual(got,
+		[]string{"gitea", "mosquitto"}) {
+		t.Fatalf("a Retry, the refusal kept only in this controller, asks %v; mosquitto was settled %q",
+			got, p.Modules["mosquitto"].State)
+	}
+}
+
+// 1713/AC2: given a build that came back without a readable manifest and was not registered, when the controller takes
+// it in, then the asking plan is failed by the build's id and the refusal is kept by id, so a Retry asks it again.
+func TestARefusalWithoutAModuleFailsItsPlanByIDAndIsRetried(t *testing.T) {
+	asked := time.Date(2026, 10, 23, 11, 51, 0, 0, time.UTC)
+	id := "build-1713-ac2-ssh-client"
+	p := failedPlanOf("ssh-client", id, asked)
+	store := keptRefusals{}
+	var failed []link.BuildResult
+	takeRefused(t.Context(), store, link.BuildResult{ID: id, Commit: "6c0024e4"}, "",
+		"what came back is not a manifest", refusedOutcome{
+			built:       func(module, _ string) { t.Fatalf("a refusal without a module was taken as %q", module) },
+			failedBuild: func(r link.BuildResult) { failed = append(failed, r) },
+		})
+
+	if len(failed) != 1 || failed[0].ID != id || !strings.Contains(failed[0].Failed, "not a manifest") {
+		t.Fatalf("the asking plan was not failed by the build's id: %+v", failed)
+	}
+	if store[id] == "" {
+		t.Fatal("the refusal was not kept in the store")
+	}
+	if got := retried(&p, store, id, "", asked.Add(time.Minute)); !reflect.DeepEqual(got,
+		[]string{"gitea", "ssh-client"}) {
+		t.Fatalf("a Retry asks %v; ssh-client was settled %q", got, p.Modules["ssh-client"].State)
+	}
+}
+
+// 1713/AC3: given a refused build kept by id, when the records settle a building plan's tier, then the module and the
+// plan are failed, never built: the plan never reports done for it.
+func TestAPlanNeverSettlesARefusedBuildBuilt(t *testing.T) {
+	asked := time.Date(2026, 10, 23, 11, 51, 0, 0, time.UTC)
+	id := "build-1713-ac3-mesh-build-go"
+	takeRefused(t.Context(), keptRefusals{}, link.BuildResult{ID: id}, "mesh-build-go", "not registered",
+		refusedOutcome{built: func(string, string) {}, failedBuild: func(link.BuildResult) {}})
+	p := inventory.Plan{ID: "plan-1713-ac3", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-go"}},
+		Modules: map[string]*inventory.PlanModule{"mesh-build-go": {State: "asked", AskedAt: &asked, Build: id}}}
+	byID := map[string]inventory.Build{id: {ID: id, Module: "mesh-build-go", At: asked.Add(time.Minute)}}
+	overlayRefusals(nil, byID, refusalsKeptFor([]string{id}))
+	settleFromRecords(&p, p.Tiers[0], nil, byID)
+	if s := p.Modules["mesh-build-go"]; s.State != "failed" || p.State != inventory.PlanFailed {
+		t.Fatalf("a refused build settled as %q, the plan %q", s.State, p.State)
+	}
+}

cmd/mesh-controller/walk_resumed_silence_1886_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Issue 1886, the resume of R4e (plan-1792707174627079192): tier 0 was sent at 00:23:28, the walk was stopped at
+// 02:01 and resumed at 07:43, and tier 1 was sent at 07:49:54. 35 seconds later the gate named ace silent "within
+// 15m0s", and at 07:51:30 shanks, in the same second shanks reported it applied: the plan failed as lost. The walk
+// judged each machine on every send it had made it, tier 0's of 00:23 too, narrowed to every module the walk carried
+// there — so tier 0's send was held against the templates of tier 1's modules it never carried, and silence was
+// measured from 00:23. A machine is judged on the walk's last send to it: that send carries its whole declaration.
+
+func r4eResumed() (inventory.Plan, []inventory.Reported, inventory.ProvLookup, time.Time) {
+	tier0 := time.Date(2026, 10, 22, 22, 23, 28, 0, time.UTC)
+	tier1 := time.Date(2026, 10, 23, 5, 49, 54, 0, time.UTC)
+	now := time.Date(2026, 10, 23, 5, 51, 30, 0, time.UTC)
+	t0, t1 := tier0, tier1
+	p := inventory.Plan{ID: "plan-1792707174627079192", State: inventory.PlanRolling, Tier: 1,
+		Tiers: [][]string{{"pacman"}, {"mesh-build-image"}, {"gitea"}},
+		Modules: map[string]*inventory.PlanModule{
+			"pacman": {State: "built", SentAt: &t0, Rest: map[string]inventory.SentDeclaration{
+				"shanks": {Digest: "d0-shanks", Sequence: 900, At: tier0,
+					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}},
+				"ace": {Digest: "d0-ace", Sequence: 1100, At: tier0,
+					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}}},
+			"mesh-build-image": {State: "built", SentAt: &t1, Rest: map[string]inventory.SentDeclaration{
+				"shanks": {Digest: "d1-shanks", Sequence: 971, At: tier1,
+					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}},
+				"ace": {Digest: "d1-ace", Sequence: 1212, At: tier1,
+					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}}},
+			"gitea": {}}}
+	shanksAt := now
+	aceAt := time.Date(2026, 10, 23, 0, 57, 50, 0, time.UTC)
+	reports := []inventory.Reported{
+		// shanks reported on tier 1's send in the second the gate looked.
+		{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &shanksAt, Declared: "d1-shanks", Current: true,
+			ReportedSequence: 971},
+		// ace's last report is from before tier 1's send, on a declaration sequenced after tier 0's.
+		{Node: "ace", Outcome: inventory.OutcomeApplied, At: &aceAt, Declared: "d-ace-0257", ReportedSequence: 1150},
+	}
+	says := map[string]inventory.ProvReport{
+		"shanks/pacman":           {Status: inventory.ProvApplied, Templates: "pac-1", At: now},
+		"shanks/mesh-build-image": {Status: inventory.ProvApplied, Templates: "img-new", At: now},
+		"ace/pacman":              {Status: inventory.ProvApplied, Templates: "pac-1", At: aceAt},
+		// ace's client had not placed the old image's files (issue 1884), and has not yet the new one's.
+		"ace/mesh-build-image": {Status: inventory.ProvWaiting, Templates: "img-old", At: aceAt},
+	}
+	prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
+	return p, reports, prov, now
+}
+
+func TestAResumedWalkJudgesEachMachineOnItsLastSend(t *testing.T) {
+	p, reports, prov, now := r4eResumed()
+	waiting, failed, silent := walkReports(p, reports, now, prov)
+	if len(silent) != 0 {
+		t.Fatalf("1886: %v named silent within %s of tier 1's send, from tier 0's send of 00:23", silent, applySilentAfter)
+	}
+	if len(failed) != 0 || !slices.Equal(waiting, []string{"ace"}) {
+		t.Fatalf("1886: waiting %v, failed %v; want ace waited for (sent 96 s ago) and shanks reported", waiting, failed)
+	}
+	// The gate the plan failed at: mesh-build-image runs on ace and shanks, which tier 2 needs.
+	needs := []gateNeed{{module: "mesh-build-image", running: []string{"ace", "shanks"},
+		since: func(string) time.Time { return *p.Modules["mesh-build-image"].SentAt }}}
+	w, quiet, lost := settleTierGate(needs, reports, silentNodes(p, reports, now, prov))
+	if lost || len(quiet) != 0 || !slices.Equal(w, []string{"mesh-build-image on ace"}) {
+		t.Fatalf("1886: the gate says waiting %v, silent %v, lost %v; want mesh-build-image on ace waited for", w, quiet, lost)
+	}
+}
+
+// The machine silent on the walk's last send to it is still silent once the bound has passed since that send.
+func TestAMachineSilentOnItsLastSendIsStillSilent(t *testing.T) {
+	p, reports, prov, now := r4eResumed()
+	later := now.Add(applySilentAfter + time.Minute)
+	_, _, silent := walkReports(p, reports, later, prov)
+	if !slices.Equal(silent, []string{"ace"}) {
+		t.Fatalf("1886: silent %v past the bound from tier 1's send; want ace", silent)
+	}
+}
+
+// A walk failed only because every machine its gate waited on was silent, whose tiers are all sent, is carried on by
+// `plans retry` once those machines reported (issue 1886): it failed on no build and no rollout, so the retry of failed
+// builds and of stopped rollouts refused it with "nothing in tier 1 … failed to build or stopped rolling out".
+func r4eLost() inventory.Plan {
+	p, _, _, _ := r4eResumed()
+	p.State = inventory.PlanFailed
+	p.Note = "tier 1 built; " + silentGateWords + " 15m0s: shanks"
+	// mesh-tools is built and recorded, never sent, as R4e's was.
+	p.Tiers[1] = append(p.Tiers[1], "mesh-tools")
+	p.Modules["mesh-tools"] = &inventory.PlanModule{State: "built"}
+	return p
+}
+
+func TestRetryCarriesOnAWalkLostToSilenceOnceItsMachinesReported(t *testing.T) {
+	p, reports, prov, now := r4eResumed()
+	p = r4eLost()
+	if !lostToSilence(p) {
+		t.Fatalf("1886: the plan R4e failed as is not read as lost to silence: %q", p.Note)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
+		t.Fatalf("1886: a walk lost to silence was refused a retry: %v", err)
+	}
+	reports[1].Current, reports[1].Declared, reports[1].ReportedSequence = true, "d1-ace", 1212
+	reports[1].At = &now
+	says := map[string]inventory.ProvReport{}
+	for _, n := range []string{"ace", "shanks"} {
+		says[n+"/pacman"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "pac-1", At: now}
+		says[n+"/mesh-build-image"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "img-new", At: now}
+	}
+	prov = func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
+	said, err := resumeLost(&p, silentNodes(p, reports, now.Add(time.Hour), prov))
+	if err != nil {
+		t.Fatalf("1886: the resume of a walk whose machines reported refused: %v", err)
+	}
+	if p.State != inventory.PlanBuilding || p.Tier != 1 {
+		t.Fatalf("1886: resumed walk is %s at tier %d; want building at tier 1, for the next look to pass its gate", p.State, p.Tier)
+	}
+	if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(said, "resumed at tier 2 of 3") {
+		t.Fatalf("1886: note %q, said %q", p.Note, said)
+	}
+}
+
+// Every machine its tier was sent to still silent: the gate would fail it again, so the retry says so instead.
+func TestRetryRefusesAWalkLostToSilenceWhileEveryMachineIsStillSilent(t *testing.T) {
+	p := r4eLost()
+	if _, err := resumeLost(&p, map[string]bool{"ace": true, "shanks": true}); err == nil || !strings.Contains(err.Error(), "ace, shanks") {
+		t.Fatalf("1886: a walk whose machine is still silent was resumed: %v", err)
+	}
+	if p.State != inventory.PlanFailed {
+		t.Fatalf("1886: a refused resume changed the plan to %s", p.State)
+	}
+}
+
+// A walk lost to silence with something of its tier never sent is not this: its send is the look's to make.
+func TestALostWalkWithATierUnsentIsNotLostToSilence(t *testing.T) {
+	p := r4eLost()
+	p.Modules["mesh-build-image"].State = "asked"
+	if lostToSilence(p) {
+		t.Fatal("1886: a tier with a module still asked was read as lost to silence")
+	}
+	p = r4eLost()
+	p.Note = "tier 1 built; mesh-build-image failed to build"
+	if lostToSilence(p) {
+		t.Fatal("1886: a plan failed by a build was read as lost to silence")
+	}
+}
+
+// A walk lost to silence is refused its resume beside a newer open plan of its repository and branch, which supersedes
+// it, and beside a related open plan: one plan at a time (ADR 0531).
+func TestRetryRefusesAWalkLostToSilenceBesideANewerOrARelatedOpenPlan(t *testing.T) {
+	p := r4eLost()
+	created := time.Date(2026, 10, 22, 22, 15, 9, 0, time.UTC)
+	p.Repository, p.Branch, p.Commit, p.Created = "novox/mesh-sdk", "main", "7aacbc16", created
+	newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
+		Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
+		Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
+	if err := retryRefusal(p, []inventory.Plan{p, newerOpen}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "plan-newer supersedes it") {
+		t.Errorf("1886: a walk lost to silence was resumed under a newer open plan of its branch: %v", err)
+	}
+	beside := inventory.Plan{ID: "plan-beside", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
+		Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-image"}},
+		Modules: map[string]*inventory.PlanModule{"mesh-build-image": {State: "asked"}}}
+	if err := retryRefusal(p, []inventory.Plan{p, beside}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("1886: a walk lost to silence was resumed beside a related open plan: %v", err)
+	}
+}

internal/builder/builder.go

@@ -242,7 +242,8 @@ func build(ctx context.Context, run Runner, publish Publisher,
 	// A fresh clone every time rather than a fetch into a tree that is already there. A build
 	// that reuses a working tree can succeed because of something a previous build left behind,
 	// and that is a build nobody can reproduce.
-	if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", repository, tree)...); err != nil {
+	if err := cloneFromForge(ctx, run, workspace, repository, tree,
+		cloneWith(credentials, "clone", "--quiet", repository, tree), say); err != nil {
 		say("clone", "FAILED: %v", err)
 		return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err)
 	}
@@ -794,7 +795,7 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
 			os.RemoveAll(dir)
 		}
 	}()
-	if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
+	if err := cloneFromForge(ctx, run, workspace, url, dir, cloneWith(credentials, "clone", "--quiet", url, dir), say); err != nil {
 		return "", fmt.Errorf("cannot clone %s: %w", url, err)
 	}
 	if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil {

internal/builder/check.go

@@ -428,7 +428,8 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	// clone clones a repository and checks out ref; readAt keeps which, by directory.
 	readAt := map[string]string{}
 	clone := func(repository, ref, dir string) error {
-		if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
+		if err := cloneFromForge(ctx, run, root, redact.redact(repository), filepath.Join(root, dir),
+			cloneWith(credentials, "clone", "--quiet", repository, dir), say); err != nil {
 			return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
 		}
 		if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {

internal/builder/clone_lookup.go

@@ -0,0 +1,114 @@
+package builder
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// A clone from the forge waits out a resolver that did not answer, for a bounded time (novox/hq issue 1773).
+//
+// **Why.** A build or a check failed now and then on "Could not resolve host: novox.internal" while cloning, between
+// clones that resolved the same name a second before and after, and the whole delivery failed with it. The lookup
+// failing is the resolver's passing fault, not the repository's: nothing reached the forge, so cloning again cannot
+// do anything twice. This is issue 457's shape (a build waits out a registry held still) applied to the name lookup.
+//
+// **Only the lookup.** Any other failure of git — authentication, a repository or a ref that is not there, a 404 —
+// fails at once, as it always did: trying it again would only make a real failure slower.
+//
+// **Bounded, and loud.** Each retry is said in the build's log as the resolver's failure, and past lookupWait the
+// build fails saying the resolver did not answer for that long.
+
+var (
+	// lookupWait is how long a clone waits for a resolver that does not answer.
+	lookupWait = 60 * time.Second
+	// lookupFirstPause is the first pause between tries; each pause doubles, up to lookupMostPause.
+	lookupFirstPause = time.Second
+)
+
+// lookupMostPause is the longest pause between two tries.
+const lookupMostPause = 10 * time.Second
+
+// lookupFailures are what git (through curl) and Go's resolver say when a name did not resolve, in lower case.
+var lookupFailures = []string{
+	"could not resolve host",
+	"temporary failure in name resolution",
+}
+
+// failedLookup is whether an error is a name lookup that failed: git's "Could not resolve host", the system's
+// "Temporary failure in name resolution", or a lookup that timed out.
+func failedLookup(err error) bool {
+	if err == nil {
+		return false
+	}
+	said := strings.ToLower(err.Error())
+	for _, f := range lookupFailures {
+		if strings.Contains(said, f) {
+			return true
+		}
+	}
+	return strings.Contains(said, "lookup ") && strings.Contains(said, "i/o timeout")
+}
+
+// cloneFromForge runs `git clone` of url into dest (git's arguments in args, run from dir), and while it fails on the
+// name lookup, clones again with a growing pause until lookupWait has passed. What a failed try left at dest is
+// removed before the next. Any other failure is returned at once.
+func cloneFromForge(ctx context.Context, run Runner, dir, url, dest string, args []string,
+	say func(step, format string, args ...any)) error {
+	_, err := run(ctx, dir, "git", args...)
+	if !failedLookup(err) {
+		return err
+	}
+	started := time.Now()
+	pause := lookupFirstPause
+	for {
+		elapsed := time.Since(started)
+		left := lookupWait - elapsed
+		if left <= 0 {
+			say("clone", "the resolver did not answer for %s, cloning %s, for %s; the clone fails",
+				urlHost(url), url, elapsed.Round(time.Second))
+			return fmt.Errorf("the resolver did not answer a lookup of %s for %s, cloning %s: %w",
+				urlHost(url), elapsed.Round(time.Millisecond), url, err)
+		}
+		wait := min(pause, left)
+		say("clone", "the resolver did not answer for %s, cloning %s; trying again in %s, for up to %s "+
+			"(novox/hq issue 1773)", urlHost(url), url, wait, lookupWait)
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped while waiting for the resolver to answer for %s: %w (last: %v)",
+				urlHost(url), ctx.Err(), err)
+		case <-time.After(wait):
+		}
+		pause = min(pause*2, lookupMostPause)
+		if err := os.RemoveAll(dest); err != nil {
+			return err
+		}
+		if _, err = run(ctx, dir, "git", args...); !failedLookup(err) {
+			if err == nil {
+				say("clone", "the resolver answered for %s after %s; the clone worked", urlHost(url),
+					time.Since(started).Round(time.Millisecond))
+			}
+			return err
+		}
+	}
+}
+
+// urlHost is the host (and port) a URL names, or the URL itself when it names none.
+func urlHost(url string) string {
+	rest := url
+	if _, after, ok := strings.Cut(url, "://"); ok {
+		rest = after
+	}
+	if i := strings.IndexAny(rest, "/?#"); i >= 0 {
+		rest = rest[:i]
+	}
+	if i := strings.LastIndex(rest, "@"); i >= 0 {
+		rest = rest[i+1:]
+	}
+	if rest == "" {
+		return url
+	}
+	return rest
+}

internal/builder/clone_lookup_test.go

@@ -0,0 +1,162 @@
+package builder
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// Issue 1773: a build or a check failed now and then on "Could not resolve host: novox.internal" while cloning from
+// the forge, between clones that resolved the same name, and its whole delivery failed with it. A clone that fails on
+// the name lookup is tried again within a bound, says each retry in the build's log as the resolver's failure, and
+// only past the bound fails, saying the resolver did not answer. Any other failure of git fails at once, as before.
+
+const lookupFailure = "exit status 128\nfatal: unable to access 'http://novox.internal:20000/novox/x.git/': " +
+	"Could not resolve host: novox.internal"
+
+// failingClones is a runner whose first n forge clones fail with failure, then clone as r does; n < 0 fails every one.
+type failingClones struct {
+	r       *recorded
+	n       int
+	failure string
+	clones  int
+}
+
+func (f *failingClones) run(ctx context.Context, dir, name string, args ...string) (string, error) {
+	if name == "git" && isACloneOf(args) {
+		f.clones++
+		if f.n < 0 || f.clones <= f.n {
+			return "", fmt.Errorf("git %s: %s", strings.Join(args, " "), f.failure)
+		}
+	}
+	return f.r.run(ctx, dir, name, args...)
+}
+
+func isACloneOf(args []string) bool {
+	for _, a := range args {
+		if a == "clone" {
+			return true
+		}
+	}
+	return false
+}
+
+// buildLog collects what a build says in its own log.
+func buildLog() (Log, func() []string) {
+	var mu sync.Mutex
+	var lines []string
+	return func(step, message string) {
+			mu.Lock()
+			defer mu.Unlock()
+			lines = append(lines, step+": "+message)
+		}, func() []string {
+			mu.Lock()
+			defer mu.Unlock()
+			return append([]string(nil), lines...)
+		}
+}
+
+const nothingToBuild = `{"module":"x","version":"1"}`
+
+// 1773/AC1: a clone failing twice on the lookup, then working, is tried again, each retry said, and the build goes on.
+func TestACloneThatFailsOnTheLookupIsTriedAgainAndTheBuildGoesOn(t *testing.T) {
+	quickLookups(t)
+	r, workspace := aRepository(t, nothingToBuild, nil)
+	f := &failingClones{r: r, n: 2, failure: lookupFailure}
+	log, said := buildLog()
+	_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, log)
+	if err != nil {
+		t.Fatalf("a clone the resolver failed twice, then answered, ended the build: %v", err)
+	}
+	if f.clones != 3 {
+		t.Errorf("the clone ran %d times, want 3 (two lookups failed, the third worked)", f.clones)
+	}
+	retries := 0
+	for _, line := range said() {
+		if strings.Contains(line, "the resolver did not answer") && strings.Contains(line, "novox.internal") {
+			retries++
+		}
+	}
+	if retries != 2 {
+		t.Errorf("the build's log said %d retries as the resolver's failure, want 2:\n%s", retries,
+			strings.Join(said(), "\n"))
+	}
+}
+
+// 1773/AC2: a lookup that keeps failing past the bound fails the build, saying the resolver did not answer.
+func TestACloneWhoseLookupNeverAnswersFailsSayingTheResolverDidNotAnswer(t *testing.T) {
+	quickLookups(t)
+	r, workspace := aRepository(t, nothingToBuild, nil)
+	f := &failingClones{r: r, n: -1, failure: lookupFailure}
+	_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err == nil {
+		t.Fatal("a clone whose lookup never answered built")
+	}
+	if !strings.Contains(err.Error(), "the resolver did not answer") ||
+		!strings.Contains(err.Error(), "http://novox.internal:20000/novox/x.git") {
+		t.Errorf("the failure does not say the resolver did not answer for the forge's URL: %v", err)
+	}
+	if f.clones < 2 {
+		t.Errorf("the clone ran %d times; a failed lookup is tried again within the bound", f.clones)
+	}
+}
+
+// 1773/AC3: any other failure of git (authentication, a missing repository) is not tried again and fails at once.
+func TestACloneFailingForAnotherReasonIsNotTriedAgain(t *testing.T) {
+	quickLookups(t)
+	for _, failure := range []string{
+		"exit status 128\nfatal: Authentication failed for 'http://novox.internal:20000/novox/x.git/'",
+		"exit status 128\nremote: Not found.\nfatal: repository 'http://novox.internal:20000/novox/x.git/' not found",
+		"exit status 128\nfatal: unable to access 'http://novox.internal:20000/novox/x.git/': The requested URL returned error: 404",
+	} {
+		r, workspace := aRepository(t, nothingToBuild, nil)
+		f := &failingClones{r: r, n: -1, failure: failure}
+		_, err := Build(context.Background(), f.run, r, "http://novox.internal:20000/novox/x.git", "", "", workspace, nil,
+			Npmrc{}, GitCredential{}, nil)
+		if err == nil || !strings.Contains(err.Error(), "cannot clone") {
+			t.Errorf("a clone failing with %q said %v", failure, err)
+		}
+		if f.clones != 1 {
+			t.Errorf("a clone failing with %q ran %d times; it is not a lookup and is not tried again", failure, f.clones)
+		}
+	}
+}
+
+// quickLookups shortens the bound and the pauses, so a test waits for milliseconds.
+func quickLookups(t *testing.T) {
+	t.Helper()
+	wasBound, wasFirst := lookupWait, lookupFirstPause
+	lookupWait, lookupFirstPause = 200*time.Millisecond, 10*time.Millisecond
+	t.Cleanup(func() { lookupWait, lookupFirstPause = wasBound, wasFirst })
+}
+
+// 1773/AC1, 1773/AC3: a check's clones are the same: a failed lookup is tried again, and the failure after it (here
+// an authentication refused) ends the check at once, not tried again.
+func TestACheckCloneTriesAFailedLookupAgain(t *testing.T) {
+	quickLookups(t)
+	clones := 0
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		if name == "git" && isACloneOf(args) {
+			clones++
+			if clones <= 2 {
+				return "", fmt.Errorf("git clone: %s", lookupFailure)
+			}
+			return "", fmt.Errorf("git clone: exit status 128\nfatal: Authentication failed")
+		}
+		return "", nil
+	}
+	_, err := Check(context.Background(), run, CheckSpec{ID: "build-1", Owner: "novox", Repo: "x", Number: 1,
+		Repository: "http://novox.internal:20000/novox/x.git", Ref: "abc"}, t.TempDir(), "registry.invalid",
+		GitCredential{}, nil)
+	if err == nil || !strings.Contains(err.Error(), "cannot clone") || !strings.Contains(err.Error(), "Authentication failed") {
+		t.Fatalf("the check said %v", err)
+	}
+	if clones != 3 {
+		t.Errorf("the check's clone ran %d times, want 3: two failed lookups, then the refusal, not tried again", clones)
+	}
+}

internal/builder/registry_wait.go

@@ -45,21 +45,37 @@ func refused(err error) bool {
 	return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
 }
 
-// waitForRegistry runs try, and while it fails because the registry at address refuses connections,
+// waitedFor is whether an error is one a build waits out: a refused connection, or a lookup of the registry's name
-// tries again with a growing pause until registryWait has passed. what names the call, for the log.
+// that the resolver did not answer (novox/hq issue 1748; failedLookup, from issue 1773, says which).
+func waitedFor(err error) bool { return refused(err) || failedLookup(err) }
+
+// waitForRegistry runs try, and while it fails because the registry at address refuses connections, or the
+// resolver does not answer a lookup of its name, tries again with a growing pause until registryWait has passed. what names the call, for the log.
 func waitForRegistry(ctx context.Context, address, what string, try func() error) error {
 	err := try()
-	if !refused(err) {
+	if !waitedFor(err) {
 		return err
 	}
 	started := time.Now()
 	pause := registryFirstPause
+	if failedLookup(err) {
+		tell("registry", "%s: the resolver did not answer a lookup of the registry's name (%v); the build waits, "+
+			"for up to %s (novox/hq issue 1748)", what, err, registryWait)
+	} else {
 		tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+
 			"the store's nightly collection runs, about a minute and a half (novox/hq issue 457)",
 			what, address, registryWait)
+	}
 	for {
 		left := registryWait - time.Since(started)
 		if left <= 0 {
+			if failedLookup(err) {
+				tell("registry", "%s: the resolver still did not answer a lookup of the registry at %s after %s; the build fails",
+					what, address, time.Since(started).Round(time.Second))
+				return fmt.Errorf("the resolver did not answer a lookup of the registry at %s for %s, so it is the "+
+					"resolver, not the registry, that is down: %w",
+					address, time.Since(started).Round(time.Millisecond), err)
+			}
 			tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address,
 				time.Since(started).Round(time.Second))
 			return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+
@@ -73,7 +89,7 @@ func waitForRegistry(ctx context.Context, address, what string, try func() error
 		case <-time.After(wait):
 		}
 		pause = min(pause*2, registryMostPause)
-		if err = try(); !refused(err) {
+		if err = try(); !waitedFor(err) {
 			// Said as it is: the registry answering is only the build going on when the call worked.
 			if err == nil {
 				tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address,

internal/builder/registry_wait_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"net"
 	"net/http"
 	"strings"
@@ -152,3 +153,67 @@ func TestAnImagePushWaitsForARegistryHeldStill(t *testing.T) {
 type errorString string
 
 func (e errorString) Error() string { return string(e) }
+
+// A resolver that does not answer for a moment (novox/hq issue 1748): a lookup of the registry's name that
+// timed out failed the build at once, though the same name resolved a second later.
+
+const lookupTimeout = "Head \"http://novox.internal:5100/v2/x/blobs/sha256:1\": dial tcp: lookup novox.internal on 10.10.0.3:53: " +
+	"read udp 10.10.0.3:44790->10.10.0.3:53: i/o timeout"
+
+type lookupTimedOut struct{}
+
+func (lookupTimedOut) Error() string { return lookupTimeout }
+
+func TestABuildWaitsOutAResolverThatDoesNotAnswerAndGoesOn(t *testing.T) { // 1748/AC1
+	waitingFor(t, 5*time.Second)
+	said := saying(t)
+	calls := 0
+	err := waitForRegistry(context.Background(), "novox.internal:5100", "HEAD /v2/x", func() error {
+		calls++
+		if calls < 3 {
+			return lookupTimedOut{}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("a lookup timing out twice failed the build: %v", err)
+	}
+	if calls != 3 {
+		t.Errorf("tried %d times, want 3", calls)
+	}
+	log := strings.Join(said(), "\n")
+	if !strings.Contains(log, "resolver did not answer") || !strings.Contains(log, "answers again") {
+		t.Errorf("the build's log does not say the resolver was waited for, and that it came back:\n%s", log)
+	}
+}
+
+func TestABuildFailsNamingTheResolverWhenItDoesNotAnswerPastTheBound(t *testing.T) { // 1748/AC2
+	waitingFor(t, 100*time.Millisecond)
+	saying(t)
+	started := time.Now()
+	err := waitForRegistry(context.Background(), "novox.internal:5100", "HEAD /v2/x", func() error { return lookupTimedOut{} })
+	if err == nil {
+		t.Fatal("a resolver that never answered let the build go on")
+	}
+	if !strings.Contains(err.Error(), "resolver did not answer") || !strings.Contains(err.Error(), "i/o timeout") {
+		t.Errorf("the failure does not name the resolver: %v", err)
+	}
+	if strings.Contains(err.Error(), "refused every connection") {
+		t.Errorf("the failure blames a refusal: %v", err)
+	}
+	if waited := time.Since(started); waited < 100*time.Millisecond {
+		t.Errorf("failed after %s, before the bound", waited)
+	}
+}
+
+func TestAnErrorThatIsNeitherARefusalNorALookupFailsAtOnce(t *testing.T) { // 1748/AC3
+	waitingFor(t, 5*time.Second)
+	calls := 0
+	err := waitForRegistry(context.Background(), "novox.internal:5100", "HEAD /v2/x", func() error {
+		calls++
+		return errors.New("401 unauthorized")
+	})
+	if err == nil || calls != 1 {
+		t.Fatalf("err %v after %d calls, want a failure at once", err, calls)
+	}
+}

internal/builderkeys/builderkeys.go

@@ -9,6 +9,7 @@ package builderkeys
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -309,3 +310,99 @@ func firstLine(s string) string {
 	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
 	return line
 }
+
+// The builder's say of what its keys hold (novox/hq issue 1886): each running builder instance keeps a key, named by its
+// node, in the builder's own announced bucket of instances, saying what the keys it writes hold. An instance that writes
+// references (ADR 0498 §1) says WritesReferences there; a builder from before R4e keeps no such key, and has no bucket.
+// The bucket is the builder's and the shape is read by the controller alone, so neither is in mesh-sdk.
+const (
+	// BuilderInstancesBucket is the builder's bucket of its running instances, `<owner module>_<name>`.
+	BuilderInstancesBucket = delivery.BuilderModule + "_instances"
+	// WritesReferences is the word an instance says when every done key it writes names where its artifacts are.
+	WritesReferences = "references"
+)
+
+// instanceSays is an instance's value in BuilderInstancesBucket: what the keys it writes hold.
+type instanceSays struct {
+	Writes []string `json:"writes"`
+}
+
+// saysReferences says whether any instance's value says it writes references. A value that does not decode says
+// nothing.
+func saysReferences(values [][]byte) bool {
+	for _, v := range values {
+		var says instanceSays
+		if json.Unmarshal(v, &says) != nil {
+			continue
+		}
+		for _, w := range says.Writes {
+			if w == WritesReferences {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// SaysReferences says whether a running builder instance says, in its bucket of instances, that it writes references. No
+// bucket (the builder from before R4e, which never made it) and no key say no; a bucket that cannot be read is an error.
+func (r Reader) SaysReferences(ctx context.Context) (bool, error) {
+	kv, err := r.bucket(ctx, BuilderInstancesBucket)
+	if err != nil || kv == nil {
+		return false, err
+	}
+	keys, err := kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("listing the builder's instances: %w", err)
+	}
+	var values [][]byte
+	for _, key := range keys {
+		e, err := kv.Get(ctx, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return false, fmt.Errorf("the builder's instance %s: %w", key, err)
+		}
+		values = append(values, e.Value())
+	}
+	return saysReferences(values), nil
+}
+
+// WatchSays calls wake for every change of the builder's bucket of instances from now on, until ctx ends: what makes the
+// look judge an open walk again when the builder starts or stops saying it writes references (issue 1886), with no
+// timer. It says false, and watches nothing, while the bucket is not there; the caller asks again when the builder
+// writes a key, which a builder that makes the bucket does.
+func WatchSays(ctx context.Context, js jetstream.JetStream, wake func()) (bool, error) {
+	kv, err := js.KeyValue(ctx, BuilderInstancesBucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("the bucket %s: %w", BuilderInstancesBucket, err)
+	}
+	w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
+	if err != nil {
+		return false, fmt.Errorf("watching %s: %w", BuilderInstancesBucket, err)
+	}
+	go func() {
+		defer w.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case e, ok := <-w.Updates():
+				if !ok {
+					return
+				}
+				if e != nil {
+					wake() // a put and a delete both change what the builder says
+				}
+			}
+		}
+	}()
+	return true, nil
+}

internal/builderkeys/says_integration_test.go

@@ -0,0 +1,54 @@
+//go:build integration
+
+package builderkeys
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// 1886/AC1 and AC4 on a real bus: no bucket of instances says no and watches nothing; an instance's key saying
+// references says yes, and its put and its delete each wake the watcher, with no timer.
+func TestTheBuildersSayIsReadAndWatchedOnTheBus(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	js := aBus(t)
+	r := Reader{JS: js}
+	if says, err := r.SaysReferences(ctx); err != nil || says {
+		t.Fatalf("no bucket: %v %v", says, err)
+	}
+	if on, err := WatchSays(ctx, js, func() {}); err != nil || on {
+		t.Fatalf("no bucket is watched: %v %v", on, err)
+	}
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: BuilderInstancesBucket, History: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	woken := make(chan struct{}, 4)
+	if on, err := WatchSays(ctx, js, func() { woken <- struct{}{} }); err != nil || !on {
+		t.Fatalf("the bucket is not watched: %v %v", on, err)
+	}
+	wake := func(what string) {
+		t.Helper()
+		select {
+		case <-woken:
+		case <-ctx.Done():
+			t.Fatalf("%s woke nothing", what)
+		}
+	}
+	put(t, kv, "shanks", []byte(`{"writes":["references"]}`))
+	wake("a put")
+	if says, err := r.SaysReferences(ctx); err != nil || !says {
+		t.Fatalf("an instance saying references: %v %v", says, err)
+	}
+	if err := kv.Delete(ctx, "shanks"); err != nil {
+		t.Fatal(err)
+	}
+	wake("a delete")
+	if says, err := r.SaysReferences(ctx); err != nil || says {
+		t.Fatalf("no instance left: %v %v", says, err)
+	}
+}

internal/builderkeys/says_test.go

@@ -0,0 +1,24 @@
+package builderkeys
+
+import "testing"
+
+// 1886/AC1: a builder that keeps no say, or a say without the word, is a builder that does not write references; one
+// instance that says it is enough, since it builds every key done without a reference again (ADR 0498 §8).
+func TestTheBuilderSaysReferencesOnlyWhenAnInstanceSaysTheWord(t *testing.T) {
+	cases := []struct {
+		name   string
+		values [][]byte
+		want   bool
+	}{
+		{"no instance", nil, false},
+		{"an instance saying nothing", [][]byte{[]byte(`{"writes":[]}`)}, false},
+		{"a value not read", [][]byte{[]byte(`{not json`)}, false},
+		{"another word", [][]byte{[]byte(`{"writes":["digests"]}`)}, false},
+		{"one instance says it", [][]byte{[]byte(`{"writes":["digests"]}`), []byte(`{"v":1,"writes":["references"]}`)}, true},
+	}
+	for _, c := range cases {
+		if got := saysReferences(c.values); got != c.want {
+			t.Errorf("%s: says references %v, want %v", c.name, got, c.want)
+		}
+	}
+}

internal/identity/fortest.go

@@ -37,8 +37,7 @@ func ForTest(t *testing.T) *Identity {
 	}
 	store.CreateMigratedForTest(t, admin, name, migrations)
 
-	cut := strings.LastIndex(admin, "/")
+	t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
-	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
 
 	ident, err := Open(t.Context())
 	if err != nil {

internal/identity/identity_test.go

@@ -7,61 +7,19 @@ import (
 	"context"
 	"crypto/ed25519"
 	"errors"
-	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/jackc/pgx/v5"
-	"github.com/novox/mesh-controller/internal/store"
 )
 
+// fresh is the package's own name for ForTest: one fixture for every store test of identity, so the way
+// a test's database is made and addressed is kept true in one place (novox/hq issue 1889: this copy
+// dropped the store address's query and reached no server on the checker's socket-addressed store).
 func fresh(t *testing.T) *Identity {
 	t.Helper()
-	admin := os.Getenv("MESH_TEST_POSTGRES")
+	return ForTest(t)
-	if admin == "" {
-		t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one")
-	}
-	name := fmt.Sprintf("ident_%d", time.Now().UnixNano()%10_000_000)
-
-	conn, err := pgx.Connect(t.Context(), admin)
-	if err != nil {
-		t.Fatalf("cannot reach the test PostgreSQL: %v", err)
-	}
-	if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
-		t.Fatalf("cannot create %s: %v", name, err)
-	}
-	conn.Close(t.Context())
-
-	cut := strings.LastIndex(admin, "/")
-	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
-
-	ident, err := Open(t.Context())
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(func() {
-		ident.Close()
-		c, err := pgx.Connect(context.Background(), admin)
-		if err != nil {
-			return
-		}
-		defer c.Close(context.Background())
-		_, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)")
-	})
-	if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
-		t.Fatal(err)
-	}
-	migrations, err := Migrations()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
-		t.Fatal(err)
-	}
-	return ident
 }
 
 func TestNoKeyIsAnErrorRatherThanAnEmptyKey(t *testing.T) {

internal/inventory/chain_rollouts.go

@@ -675,6 +675,12 @@ func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
 		since = old.Created
 	}
 	wasOpen := old.State == PlanBuilding || old.State == PlanRolling
+	if !wasOpen && p.Open() {
+		// **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept,
+		// the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s"
+		// seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once.
+		return now, nil
+	}
 	if !wasOpen || (old.Tier == p.Tier && p.Open()) {
 		return since, nil // still in the tier, or already ended
 	}

internal/inventory/fortest.go

@@ -42,8 +42,7 @@ func ForTest(t *testing.T) *Inventory {
 	store.CreateMigratedForTest(t, admin, name, migrations)
 	busPasswordCostForTest.Store(int32(bcrypt.MinCost))
 
-	cut := strings.LastIndex(admin, "/")
+	t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
-	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
 
 	inv, err := Open(t.Context())
 	if err != nil {
@@ -113,8 +112,7 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
 	}
 	store.CreateMigratedForTest(t, admin, name, before)
 	busPasswordCostForTest.Store(int32(bcrypt.MinCost))
-	cut := strings.LastIndex(admin, "/")
+	t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
-	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
 	inv, err := Open(t.Context())
 	if err != nil {
 		t.Fatal(err)

internal/inventory/tier_entered_resume_1886_test.go

@@ -0,0 +1,32 @@
+package inventory
+
+import (
+	"testing"
+	"time"
+)
+
+// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the
+// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s"
+// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at
+// enters that tier anew: its time in the tier runs from the resume.
+func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) {
+	entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC)
+	resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC)
+	old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}},
+		Created: entered.Add(-2 * time.Hour), TierEntered: entered}
+	p := *old
+	p.State = PlanBuilding
+	got, d := tierLeftOf(old, p, resumed)
+	if !got.Equal(resumed) || d != nil {
+		t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed)
+	}
+	// A walk still in its tier keeps its entry; one that ended keeps it too.
+	open := *old
+	open.State = PlanRolling
+	if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) {
+		t.Fatalf("an open walk still in its tier was stamped anew: %s", got)
+	}
+	if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) {
+		t.Fatalf("a failed walk saved again was stamped anew: %s", got)
+	}
+}

internal/licences/fortest.go

@@ -40,8 +40,7 @@ func ForTest(t *testing.T) *Licences {
 	}
 	store.CreateMigratedForTest(t, admin, name, migrations)
 
-	cut := strings.LastIndex(admin, "/")
+	t.Setenv(store.Variable(Name), store.DatabaseForTest(admin, name))
-	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
 
 	held, err := Open(t.Context())
 	if err != nil {

internal/store/fortest.go

@@ -110,12 +110,7 @@ func raiseTemplate(ctx context.Context, admin, name string, key int64, migration
 	if _, err := conn.Exec(ctx, "create database "+name); err != nil {
 		return err
 	}
-	cut := strings.LastIndex(admin, "/")
+	pool, err := pgxpool.New(ctx, DatabaseForTest(admin, name))
-	rest := ""
-	if q := strings.Index(admin[cut:], "?"); q >= 0 {
-		rest = admin[cut+q:]
-	}
-	pool, err := pgxpool.New(ctx, admin[:cut]+"/"+name+rest)
 	if err != nil {
 		return err
 	}
@@ -136,3 +131,19 @@ func raiseTemplate(ctx context.Context, admin, name string, key int64, migration
 	}
 	return nil
 }
+
+// DatabaseForTest is the address of the database name on the same test PostgreSQL that admin reaches: admin
+// with its database replaced and its query kept.
+//
+// **The query is kept, never replaced** (novox/hq issue 1889): the checker's integration store answers on a
+// unix socket named in the query (`?host=<socket directory>&sslmode=disable`), and an address that drops it
+// reaches pgx's default socket, where no server is, so every store test waited out its readiness bound and
+// failed. Every store fixture builds its database's address here, so none can drop it again.
+func DatabaseForTest(admin, name string) string {
+	cut := strings.LastIndex(admin, "/")
+	query := ""
+	if q := strings.Index(admin[cut:], "?"); q >= 0 {
+		query = admin[cut+q:]
+	}
+	return admin[:cut] + "/" + name + query
+}

internal/store/fortest_test.go

@@ -0,0 +1,25 @@
+package store
+
+import "testing"
+
+// 1889/AC1: the checker's integration store answers on a unix socket named in the address's query
+// (mesh-build-go's store). A test database's address that drops the query reaches pgx's default socket,
+// where no server is, and every store test waits out its 20 s and fails.
+func TestATestDatabaseIsOnTheSocketItsAdminAddressNames(t *testing.T) {
+	admin := "postgres://postgres@/postgres?host=%2Fwork%2Frun-1%2Fpg%2Fsocket&sslmode=disable"
+	got := DatabaseForTest(admin, "ident_1_x")
+	want := "postgres://postgres@/ident_1_x?host=%2Fwork%2Frun-1%2Fpg%2Fsocket&sslmode=disable"
+	if got != want {
+		t.Fatalf("the test database's address is %q, want %q", got, want)
+	}
+}
+
+// 1889/AC2: the address `make check` gives (TCP, with sslmode) builds what the fixtures always built.
+func TestATestDatabaseOnTCPIsAddressedAsBefore(t *testing.T) {
+	admin := "postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable"
+	got := DatabaseForTest(admin, "inv_2_y")
+	want := "postgres://postgres:check@127.0.0.1:55566/inv_2_y?sslmode=disable"
+	if got != want {
+		t.Fatalf("the test database's address is %q, want %q", got, want)
+	}
+}

internal/store/live_test.go

@@ -55,7 +55,7 @@ func freshStore(t *testing.T) *Store {
 	}
 	conn.Close(t.Context())
 
-	t.Setenv(Variable("testing"), replaceDatabase(dsn, name))
+	t.Setenv(Variable("testing"), DatabaseForTest(dsn, name))
 	s, err := Open(t.Context(), "testing")
 	if err != nil {
 		t.Fatal(err)
@@ -75,15 +75,6 @@ func freshStore(t *testing.T) *Store {
 	return s
 }
 
-func replaceDatabase(dsn, name string) string {
-	cut := strings.LastIndex(dsn, "/")
-	rest := ""
-	if q := strings.Index(dsn[cut:], "?"); q >= 0 {
-		rest = dsn[cut+q:]
-	}
-	return dsn[:cut] + "/" + name + rest
-}
-
 func TestTheSchemaIsAppliedAndRecorded(t *testing.T) {
 	s := freshStore(t)
 	migrations := []Migration{{Number: 1, Name: "people", SQL: "create table person (id int)", Checksum: "a"}}