Back off before standing for the controller lease after a loss, and while the bus does not answer (issue 906)

A controller that lost its lease to a stalled bus exited and was started
again five seconds later, asserting every bucket and consumer on the bus
that had just stalled; a start that could not assert the lease bucket
exited at once, into the same loop. A candidate now waits one key's age
after the newest epoch lost in the last ten minutes, doubled per further
loss, at most two minutes, before it touches the bus; and retries its
stand in-process from 5 s doubling to 60 s instead of exiting.

cmd/mesh-controller/acting.go

@@ -30,7 +30,9 @@ import (
 //     closes at once and the process exits, so its service manager restarts it as a candidate (serve,
 //     in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the
 //     lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal
-//     made opens the gate again.
+//     made opens the gate again. A candidate backs off before it stands (novox/hq issue 906, lease_backoff.go):
+//     after an epoch lost lately, and while its bus does not answer it, in its own process rather than in a
+//     restart every five seconds.
 //   - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose
 //     controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the
 //     lease: it is the same mesh's word, composed and sent under the store's hold of each machine like
@@ -72,6 +74,9 @@ type actor struct {
 	// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
 	// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
 	renewEvery, poll time.Duration
+	// lossBase, when set, is the backoff after a lost epoch in place of the key's age: a test's (novox/hq
+	// issue 906).
+	lossBase time.Duration
 }
 
 // theLease is this process's standing to the lease.
@@ -211,48 +216,73 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
 	a.mu.Lock()
 	a.serving = true
 	a.mu.Unlock()
-	// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
-	js, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
-	if err != nil {
-		return nil, fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
-			"lease: %w", broker.BareAddress(address), err)
-	}
-	api, err := jetstream.New(js.Conn())
-	if err != nil {
-		js.Close()
-		return nil, err
-	}
-	asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
-	err = broker.EnsureLeaseBucket(asserting, api)
-	cancel()
-	if err != nil {
-		js.Close()
-		return nil, err
-	}
 	say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
-	l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
-		RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
-			a.mu.Lock()
-			defer a.mu.Unlock()
-			a.reset = time.Now()
-			a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
-				"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
-				"no machine refuses the next epoch", was, floor, floor)
-		}})
+	// **Backing off before standing** (novox/hq issue 906, lease_backoff.go): after an epoch lost lately, before
+	// this process touches the bus; and while the bus does not answer it, here rather than in a restart.
+	if wait := a.backoffAfterLosses(ctx, inv, say); wait > 0 {
+		if err := sleepOrDone(ctx, wait); err != nil {
+			return nil, err
+		}
+	}
+	var (
+		js         *broker.JetStream
+		l          *lease.Lease
+		epoch      uint64
+		unwritable error
+	)
+	err = standUp(ctx, sleepOrDone, say, func(ctx context.Context) error {
+		unwritable = nil
+		if js == nil {
+			// Its own connection, held as long as the lease, and named so (novox/hq issue 327).
+			dialled, err := broker.Dial(address, nats.Name(broker.ConnectionName+" lease"))
+			if err != nil {
+				return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it to take the "+
+					"lease: %w", broker.BareAddress(address), err)
+			}
+			js = dialled
+		}
+		api, err := jetstream.New(js.Conn())
+		if err != nil {
+			return err
+		}
+		asserting, cancel := context.WithTimeout(ctx, 10*time.Second)
+		err = broker.EnsureLeaseBucket(asserting, api)
+		cancel()
+		if err != nil {
+			return err
+		}
+		l, err = lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
+			RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
+				a.mu.Lock()
+				defer a.mu.Unlock()
+				a.reset = time.Now()
+				a.resetSaid = fmt.Sprintf("the lease bucket was at revision %d with epoch %d already issued: it was "+
+					"raised again from nothing (a bus whose data was replaced), and its revisions were moved past %d so "+
+					"no machine refuses the next epoch", was, floor, floor)
+			}})
+		if err != nil {
+			return err
+		}
+		epoch, err = l.Take(ctx)
+		if errors.Is(err, lease.ErrUnwritable) {
+			unwritable = err
+			return nil
+		}
+		// Any other error: whether another controller acts cannot be told, so this one does not act, and tries
+		// again.
+		return err
+	})
 	if err != nil {
-		js.Close()
+		if js != nil {
+			js.Close()
+		}
 		return nil, err
 	}
 	gone := make(chan struct{})
-	epoch, err := l.Take(ctx)
-	switch {
+	switch err := unwritable; {
 	case ctx.Err() != nil:
 		js.Close()
 		return nil, ctx.Err()
-	case err != nil && !errors.Is(err, lease.ErrUnwritable):
-		// Whether another controller acts cannot be told: this one does not act, and exits to try again.
-		js.Close()
-		return nil, err
 	case err != nil:
 		// Nobody holds it and the bus will not let it be written: unleased, said, tried again (see above).
 		a.mu.Lock()

cmd/mesh-controller/lease_backoff.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Backing off before standing (novox/hq issue 906).
+//
+// A controller loses its lease when its bus stops answering for longer than the key's age: on the control-node,
+// a bus whose disks are saturated has taken 30 to 42 seconds to answer (2026-10-23 05:52). The process then
+// exits, as ADR 0229 rule 4 says, and its service manager starts it again five seconds later. **A start is the
+// heaviest thing a controller asks of the bus** — every bucket, stream and consumer asserted again — and a start
+// that cannot assert the lease's bucket used to exit at once and be started again five seconds later, on the
+// same overloaded bus, nine times in twelve minutes (2026-10-16). So a candidate backs off twice:
+//
+//   - **After a loss**, before it touches the bus: from the newest epoch lost in the last ten minutes (the
+//     controller's store says it, rule 7), it waits one key's age, doubled for every further loss in that
+//     window, at most two minutes. A single loss costs a key's age; a controller losing it again and again
+//     stands less and less often.
+//   - **While the bus does not answer it**, in its own process: asserting the lease's bucket, opening it or
+//     reading who holds it is tried again after 5 s, doubling to at most 60 s, instead of exiting into a restart
+//     every five seconds. Nothing acts meanwhile: the process holds no lease, and the gate refuses every act.
+
+const (
+	// lossWindow is how far back lost epochs count towards the backoff.
+	lossWindow = 10 * time.Minute
+	// lossMost is the longest a candidate waits after losses.
+	lossMost = 2 * time.Minute
+	// standFirst and standMost are the waits between tries of a candidate the bus does not answer.
+	standFirst = 5 * time.Second
+	standMost  = time.Minute
+)
+
+// lossBackoff is how long a candidate waits now before it stands, after the epochs it read: one key's age from
+// the newest loss in the window, doubled for every further loss in it, at most lossMost.
+func lossBackoff(epochs []inventory.Epoch, now time.Time) time.Duration {
+	return lossBackoffFrom(epochs, now, broker.LeaseTTL)
+}
+
+func lossBackoffFrom(epochs []inventory.Epoch, now time.Time, base time.Duration) time.Duration {
+	lost, newest := lostSince(epochs, now)
+	if lost == 0 {
+		return 0
+	}
+	delay := base
+	for i := 1; i < lost && delay < lossMost; i++ {
+		delay *= 2
+	}
+	delay = min(delay, lossMost)
+	return max(newest.Add(delay).Sub(now), 0)
+}
+
+// standUp runs try until it answers nil, waiting standFirst between tries and doubling to standMost; it answers
+// ctx's error once ctx ends.
+func standUp(ctx context.Context, sleep func(context.Context, time.Duration) error, say func(string, ...any),
+	try func(context.Context) error) error {
+	wait := standFirst
+	for {
+		err := try(ctx)
+		if err == nil {
+			return nil
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		say("this controller cannot stand for the lease yet: %v — it tries again in %s, in this process, rather "+
+			"than start again on a bus that does not answer (novox/hq issue 906)", err, wait)
+		if err := sleep(ctx, wait); err != nil {
+			return err
+		}
+		wait = min(wait*2, standMost)
+	}
+}
+
+// backoffAfterLosses is how long this candidate waits before it stands, read from the epochs its store kept,
+// and said. A store that cannot be read makes it wait nothing: the backoff spares the bus, and never keeps a
+// controller from standing.
+func (a *actor) backoffAfterLosses(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) time.Duration {
+	now := time.Now()
+	reading, cancel := context.WithTimeout(ctx, 10*time.Second)
+	epochs, err := inv.EpochsSince(reading, now.Add(-lossWindow))
+	cancel()
+	if err != nil {
+		say("whether the lease was lost lately cannot be read, so this controller stands without backing off: %v", err)
+		return 0
+	}
+	base := a.lossBase
+	if base <= 0 {
+		base = broker.LeaseTTL
+	}
+	wait := lossBackoffFrom(epochs, now, base)
+	if wait > 0 {
+		lost, _ := lostSince(epochs, now)
+		say("the controller lease was lost lately (%d time(s) in the last %s): this controller backs off %s "+
+			"before it stands, so its start does not add to the load that lost it (novox/hq issue 906)",
+			lost, lossWindow, wait.Round(time.Second))
+	}
+	return wait
+}
+
+// lostSince is how many epochs were lost in the window before now, and when the newest of them was.
+func lostSince(epochs []inventory.Epoch, now time.Time) (lost int, newest time.Time) {
+	for _, e := range epochs {
+		if e.Ended == nil || e.How != inventory.EpochLost || now.Sub(*e.Ended) > lossWindow {
+			continue
+		}
+		lost++
+		if e.Ended.After(newest) {
+			newest = *e.Ended
+		}
+	}
+	return lost, newest
+}

cmd/mesh-controller/lease_backoff_test.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A controller that lost its lease to a slow bus backs off before it stands again (novox/hq issue 906): it
+// does not add its start — every bucket and consumer asserted again — to the load that lost it the lease.
+
+func lostAt(epoch uint64, ended time.Time, how string) inventory.Epoch {
+	return inventory.Epoch{Epoch: epoch, Instance: "controller@anchor", Taken: ended.Add(-time.Hour), Ended: &ended,
+		How: how}
+}
+
+// 906/AC1: given epochs lost in the last ten minutes, when a candidate starts, then it waits from the newest
+// loss one key's age, doubled for each further loss, at most two minutes; nothing else makes it wait.
+func TestACandidateAfterALostLeaseBacksOff(t *testing.T) {
+	now := time.Date(2026, 10, 23, 5, 52, 40, 0, time.UTC)
+	for _, c := range []struct {
+		name   string
+		epochs []inventory.Epoch
+		want   time.Duration
+	}{
+		{"nothing lost", nil, 0},
+		{"one held, one given back", []inventory.Epoch{
+			{Epoch: 3, Taken: now.Add(-time.Minute)},
+			lostAt(2, now.Add(-2*time.Second), inventory.EpochReleased),
+		}, 0},
+		{"one lost five seconds ago", []inventory.Epoch{lostAt(7, now.Add(-5*time.Second), inventory.EpochLost)},
+			10 * time.Second},
+		{"one lost long ago enough", []inventory.Epoch{lostAt(7, now.Add(-20*time.Second), inventory.EpochLost)}, 0},
+		{"three lost, the newest a second ago", []inventory.Epoch{
+			lostAt(9, now.Add(-time.Second), inventory.EpochLost),
+			lostAt(8, now.Add(-3*time.Minute), inventory.EpochLost),
+			lostAt(7, now.Add(-6*time.Minute), inventory.EpochLost),
+		}, 59 * time.Second},
+		{"six lost: at most two minutes", []inventory.Epoch{
+			lostAt(12, now, inventory.EpochLost), lostAt(11, now.Add(-time.Minute), inventory.EpochLost),
+			lostAt(10, now.Add(-2*time.Minute), inventory.EpochLost), lostAt(9, now.Add(-3*time.Minute), inventory.EpochLost),
+			lostAt(8, now.Add(-4*time.Minute), inventory.EpochLost), lostAt(7, now.Add(-5*time.Minute), inventory.EpochLost),
+		}, 2 * time.Minute},
+		{"a loss older than ten minutes does not count", []inventory.Epoch{
+			lostAt(9, now.Add(-time.Second), inventory.EpochLost),
+			lostAt(8, now.Add(-11*time.Minute), inventory.EpochLost),
+		}, 14 * time.Second},
+		{"expired is not lost", []inventory.Epoch{lostAt(9, now.Add(-time.Second), inventory.EpochExpired)}, 0},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			if got := lossBackoff(c.epochs, now); got != c.want {
+				t.Fatalf("a candidate after %d epoch(s) waits %s, want %s", len(c.epochs), got, c.want)
+			}
+		})
+	}
+}
+
+// 906/AC2: given a bus that does not answer a candidate, when it asserts the lease's bucket or reads who holds
+// it, then the candidate tries again in its own process after 5 s, doubling to at most 60 s, instead of exiting
+// to be restarted by its service manager every five seconds.
+func TestACandidateTheBusDoesNotAnswerTriesAgainWithBackoff(t *testing.T) {
+	var waited []time.Duration
+	sleep := func(ctx context.Context, d time.Duration) error { waited = append(waited, d); return nil }
+	tries := 0
+	slow := errors.New("asserting bucket mesh-controller_lease: context deadline exceeded")
+	err := standUp(t.Context(), sleep, func(string, ...any) {}, func(context.Context) error {
+		tries++
+		if tries < 7 {
+			return slow
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("a candidate whose bus answered at the seventh try gave up: %v", err)
+	}
+	want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute,
+		time.Minute}
+	if tries != 7 || !slices.Equal(waited, want) {
+		t.Fatalf("a candidate tried %d time(s), waiting %v between; want 7, waiting %v", tries, waited, want)
+	}
+}
+
+// 906/AC2: and a candidate stopping while it waits stops, with why.
+func TestACandidateStoppingWhileItBacksOffStops(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	err := standUp(ctx, sleepOrDone, func(string, ...any) {}, func(context.Context) error {
+		return errors.New("who holds the lease cannot be read: context deadline exceeded")
+	})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("a candidate stopped while backing off answered %v", err)
+	}
+}

cmd/mesh-controller/lease_integration_test.go

@@ -152,3 +152,33 @@ func TestTwoControllersOneActs(t *testing.T) {
 		t.Fatalf("B's epoch does not say it was lost: %+v", ended)
 	}
 }
+
+// 906/AC1, on a real bus and store: a controller starting right after an epoch was lost backs off before it
+// stands — it takes the lease no sooner than the backoff after the loss, and then holds it as usual.
+func TestAControllerAfterALostLeaseBacksOffBeforeItStands(t *testing.T) {
+	url, _ := aBusForTheLease(t)
+	inv := inventory.ForTest(t)
+	ctx := t.Context()
+	// The controller before it lost epoch 5, and said so in the store, as `took` does on a loss.
+	if _, err := inv.TookEpoch(ctx, inventory.Epoch{Epoch: 5, Instance: "controller@anchor pid 1", Host: "anchor",
+		Taken: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.EndEpoch(ctx, 5, inventory.EpochLost); err != nil {
+		t.Fatal(err)
+	}
+	// The key's age stands in as a backoff of two seconds, so the case runs in seconds (issue 591).
+	const backoff = 2 * time.Second
+	a := &actor{renewEvery: lease.RenewEvery / 5, poll: lease.Poll / 10, lossBase: backoff}
+	start := time.Now()
+	if _, err := a.serveUnderTheLease(ctx, inv, url); err != nil {
+		t.Fatal(err)
+	}
+	defer a.release()
+	if waited := time.Since(start); waited < backoff-300*time.Millisecond {
+		t.Fatalf("a controller took the lease %s after an epoch was lost, without backing off %s", waited, backoff)
+	}
+	if epoch, err := a.epoch(ctx); err != nil || epoch <= 5 {
+		t.Fatalf("after backing off the controller holds epoch %d (%v), want one above the lost 5", epoch, err)
+	}
+}

cmd/mesh-controller/plan_retry.go

@@ -203,6 +203,21 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
 		return oneWalkAtATime(p, plans, onPath)
 	}
 	stopped := stoppedRollouts(p)
+	// **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is
+	// asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send.
+	if len(stopped) == 0 && stoppedByHandWaiting(p) {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		for _, m := range holdingABuild(p) {
+			if q, found := newerPlanFor(m, p, plans); found {
+				return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+
+					"over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
+			}
+		}
+		return oneWalkAtATime(p, plans, onPath)
+	}
 	if len(stopped) == 0 {
 		return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
 			p.Tier, p.ID, p.Note)
@@ -333,6 +348,8 @@ func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inv
 
 // retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
 // that tier: what follows is the plan going on as if they had built the first time.
+// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again
+// (resumeStopped, issue 1886).
 func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	inv := open.inventory
 	release, err := inv.HoldPlans(ctx, true)
@@ -388,6 +405,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
 		return retryTierWhole(ctx, open, &p, again)
 	}
+	if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) {
+		said := resumeStopped(&p)
+		if err := inv.SavePlan(ctx, &p); err != nil {
+			return "", err
+		}
+		return said, nil
+	}
 	if len(failed) == 0 {
 		return retryRollouts(ctx, open, &p)
 	}
@@ -583,3 +607,89 @@ func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[
 	}
 	return failedIn(*p)
 }
+
+// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a
+// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left
+// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299).
+func stoppedByHandWaiting(p inventory.Plan) bool {
+	if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" ||
+		p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 {
+		return false
+	}
+	asked, unsent, unasked := waitingIn(p)
+	return asked+unsent+unasked > 0
+}
+
+// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked.
+func waitingIn(p inventory.Plan) (asked, unsent, unasked int) {
+	if p.Tier >= len(p.Tiers) {
+		return 0, 0, 0
+	}
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		switch {
+		case s == nil || s.State == "":
+			unasked++
+		case s.State == "asked":
+			asked++
+		case s.State == "built" && s.SentAt == nil:
+			unsent++
+		}
+	}
+	return asked, unsent, unasked
+}
+
+// holdingABuild is the modules of a stopped walk's current tier that hold a build of its commit, not yet sent: built and
+// not sent, or asked under a build id. Only these could put an older build over a newer plan's (review of issue 1886): a
+// module not yet asked is asked at its followed branch's head when the walk reaches it, and one asked on the builder's key
+// is settled by the look from the key of the commit it reads then. Sorted.
+func holdingABuild(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		if s == nil || s.SentAt != nil || s.Key {
+			continue
+		}
+		if s.State == "built" || (s.State == "asked" && s.Build != "") {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept but
+// build-agent's own (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the
+// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it.
+func resumeStopped(p *inventory.Plan) string {
+	// **Build-agent's own are asked of build-agent** (novox/hq ADR 0522 §6): one marked taken from a key it writes itself is
+	// set unasked, and the look asks the rest of a partly asked tier (unaskedOf), of build-agent, whatever the builder says.
+	var again []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.State == "asked" && s.Key && isBuildersOwn(m) {
+			s.State, s.Key, s.AskedAt, s.Build, s.Why = "", false, nil, "", ""
+			again = append(again, m)
+		}
+	}
+	sort.Strings(again)
+	asked, unsent, unasked := waitingIn(*p)
+	later := 0
+	for i := p.Tier + 1; i < len(p.Tiers); i++ {
+		later += len(p.Tiers[i])
+	}
+	by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy
+	waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after",
+		asked, unsent, unasked, later)
+	if len(again) > 0 {
+		waits += "; " + strings.Join(again, ", ") + " set unasked, for the look to ask of build-agent (its own, ADR 0522 §6)"
+	}
+	p.State = inventory.PlanBuilding
+	goesOnAgain(p)
+	p.Delivery.Kept = nil
+	p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits)
+	return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+
+		"on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits)
+}

cmd/mesh-controller/plan_retry_stopped_test.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next
+// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds
+// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do
+// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on.
+func TestRetryResumesAWalkStoppedByHand(t *testing.T) {
+	at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
+	sent := at.Add(-time.Hour)
+	p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
+		Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
+		Note: "stopped by a person at tier 1: issue 1886",
+		Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886",
+			Kept: []inventory.PlanKept{{Module: "mesh-controller"}}},
+		Modules: map[string]*inventory.PlanModule{}}
+	tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"}
+	for _, m := range tier0 {
+		p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+	}
+	var tier1 []string
+	for _, m := range []string{"build-agent", "mesh-tools"} {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "built", AskedAt: &sent, BuiltAt: &sent, Build: "build-" + m}
+	}
+	onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker",
+		"mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"}
+	for _, m := range onKeys {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m}
+	}
+	tier2 := []string{"gitea", "mesh-catalog", "mongodb"}
+	for _, m := range tier2 {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	p.Tiers = [][]string{tier0, tier1, tier2}
+
+	if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
+		t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err)
+	}
+	said := resumeStopped(&p)
+	if p.State != inventory.PlanBuilding {
+		t.Errorf("resumed walk is %s, wanted building", p.State)
+	}
+	if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 {
+		t.Errorf("resumed walk still says it stopped: %+v", p.Delivery)
+	}
+	if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") {
+		t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note)
+	}
+	for _, m := range onKeys {
+		s := p.Modules[m]
+		if isBuildersOwn(m) {
+			// Build-agent's own are asked of build-agent whatever the builder says (ADR 0522 §6): set unasked, for the look.
+			if s.State != "" || s.Key || s.Build != "" || s.AskedAt != nil {
+				t.Errorf("%s, build-agent's own, was left on the builder's key: %+v", m, s)
+			}
+			if !slices.Contains(unaskedOf(p), m) {
+				t.Errorf("%s is not among what the look asks next: %v", m, unaskedOf(p))
+			}
+			continue
+		}
+		if s.State != "asked" || !s.Key || s.Build != "build-"+m {
+			t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s)
+		}
+	}
+	for _, w := range []string{"plan-r4e", "tier 2 of 3", "5 asked", "2 built and not sent", "3 not yet asked in it",
+		"3 not yet asked in the tiers after", "mesh-build-go, mesh-build-image, mesh-build-node set unasked"} {
+		if !strings.Contains(said, w) {
+			t.Errorf("the answer %q does not say %q", said, w)
+		}
+	}
+
+	// What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside.
+	done := p
+	done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	done.Modules = map[string]*inventory.PlanModule{}
+	for _, tier := range done.Tiers {
+		for _, m := range tier {
+			done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+		}
+	}
+	if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") {
+		t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err)
+	}
+	stopped := p
+	stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
+		Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
+		Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("a stopped walk was resumed beside a related open walk: %v", err)
+	}
+	// A newer open plan of its repository and branch supersedes it: resuming would build what that one replaced.
+	newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
+		Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
+		Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{stopped, newerOpen}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "plan-newer supersedes it") {
+		t.Errorf("a stopped walk was resumed under a newer open plan of its branch: %v", err)
+	}
+	// A newer plan holding a build of its tier not yet sent refuses it: resuming would send the older build over the newer.
+	newerBuilt := inventory.Plan{ID: "plan-agent", Repository: "novox/mesh-controller", Branch: "main", Commit: "d00dfeed",
+		Created: at, State: inventory.PlanDone, Tiers: [][]string{{"build-agent"}},
+		Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{stopped, newerBuilt}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "older build") {
+		t.Errorf("a stopped walk was resumed over a newer plan of a build it has not sent: %v", err)
+	}
+}
+
+// A newer plan holding a module the stopped walk has not yet asked does not refuse its resume (review of issue 1886): that
+// module is asked at its followed branch's head when the walk reaches it, so no older build goes over the newer. Open and
+// related, the newer walk refuses it under one walk at a time; done, it does not. One holding a build the walk has not sent
+// still refuses it.
+func TestResumeIsNotRefusedByANewerPlanOfAModuleNotYetAsked(t *testing.T) {
+	at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
+	sent := at.Add(-time.Hour)
+	p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
+		Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
+		Delivery: &inventory.PlanDelivery{Stopped: "a person", StoppedWhy: "issue 1886"},
+		Tiers:    [][]string{{"mesh-sdk"}, {"build-agent", "mesh-delivery-gate"}, {"mesh-delivery-builder", "gitea"}},
+		Modules: map[string]*inventory.PlanModule{
+			"mesh-sdk":              {State: "built", BuiltAt: &sent, SentAt: &sent},
+			"build-agent":           {State: "built", BuiltAt: &sent},
+			"mesh-delivery-gate":    {State: "asked", AskedAt: &sent, Key: true},
+			"mesh-delivery-builder": {}, "gitea": {}}}
+	newer := func(state string, holds string) inventory.Plan {
+		return inventory.Plan{ID: "plan-1792715180015327197", Repository: "novox/mesh-delivery", Branch: "main",
+			Commit: "d00dfeed", Created: at, State: state, Tiers: [][]string{{holds}},
+			Modules: map[string]*inventory.PlanModule{holds: {State: "asked"}}}
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-builder")}, ownPathOf(nil)); err != nil {
+		t.Errorf("a done newer plan of a module not yet asked refused the resume: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-gate")}, ownPathOf(nil)); err != nil {
+		t.Errorf("a done newer plan of a module on the builder's key refused the resume: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanBuilding, "mesh-delivery-builder")}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("an open newer walk related to it did not refuse the resume under one walk at a time: %v", err)
+	}
+	if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "build-agent")}, ownPathOf(nil)); err == nil ||
+		!strings.Contains(err.Error(), "older build") {
+		t.Errorf("a newer plan of a built module not yet sent did not refuse the resume: %v", err)
+	}
+}