Assemble merges in a rolling window and walk each batch once (hq ADR 0276, issue 362)

Every merge opened a walk and the next merge of the branch superseded it: two
catalogue merges 18 s apart left a walk no delivery held, and the operator
started it by hand 58 minutes later. A merge now joins the open batch, kept in
the store (migration 0089), which is cut into one walk when no merge came for
merge-window (90 s) or at merge-window-at-most (10 min): one commit per
repository, the latest of its branch, with every file the batch's merges
changed. One walk at a time; a started walk is never superseded, a waiting one
is folded into the next. The walk names every merge it answers on the wire
(delivery.merges, taken_over_by, batch). A failed walk walks its earlier merges
alone, newest first, until one is delivered. A delivery group's order becomes
tier edges inside the walk. plans shows the batch assembling; S18 and S19
bound its waits; S16 names the merges a waiting walk answers.

cmd/mesh-controller/plan_retry.go

@@ -164,12 +164,13 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
 	case p.Open():
 		return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
 	}
+
 	if len(failedIn(p)) > 0 {
 		if q, found := newerOpenPlan(p, plans); found {
 			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
 				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
 		}
-		return nil
+		return oneWalkAtATime(p, plans)
 	}
 	stopped := stoppedRollouts(p)
 	if len(stopped) == 0 {
@@ -193,6 +194,17 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
 				"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
 		}
 	}
+	return oneWalkAtATime(p, plans)
+}
+
+// oneWalkAtATime refuses a retry while another walk that started is open (novox/hq ADR 0276): a walk retried
+// beside it would be two walks at once.
+func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
+	for _, q := range plans {
+		if q.ID != p.ID && q.Open() && q.Release == nil && !q.Waiting() {
+			return fmt.Errorf("%s is open (%s): one walk at a time — retry %s once it ended", q.ID, q.Named(), p.ID)
+		}
+	}
 	return nil
 }