Give the forge's merges a durable consumer of their own: no max-deliver, no reset, a growing delay while held (issue 973)

internal/link/advisories.go

@@ -145,7 +145,7 @@ func MeshNamed(stream, name string) bool {
 		return true
 	case strings.HasPrefix(stream, "SEAT_"):
 		return strings.HasSuffix(name, "_worker")
-	case name == broker.ControllerName:
+	case name == broker.ControllerName || name == broker.MergesConsumerName:
 		return true
 	case stream == broker.EventsStream:
 		return strings.Contains(name, "_")
@@ -156,7 +156,7 @@ func MeshNamed(stream, name string) bool {
 // ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
 func ConsumerInWords(stream, name string) string {
 	switch {
-	case name == broker.ControllerName:
+	case name == broker.ControllerName || name == broker.MergesConsumerName:
 		return "the controller's consumer on " + stream
 	case stream == "NODES":
 		return "how " + name + " hears what it should be (its declaration consumer)"

internal/link/receive_nats.go

@@ -148,6 +148,24 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
 		defer func() { _ = followed.Unsubscribe() }()
 	}
 
+	// And the forge's merges, off a consumer of their own: no max-deliver and no reset, so a merge held for the
+	// plans waits for them however long (novox/hq ADR 0407 decision 3). Bound to the same channel: one loop.
+	if n.follows[KindSourceMoved] {
+		if events == nil {
+			events = make(chan *nats.Msg, Prefetch)
+		}
+		merges, err := standingBy(ctx, log.Default(), "EVENTS", func() (*nats.Subscription, error) {
+			return js.ChanSubscribe(MergedSubject, events, nats.Bind("EVENTS", broker.MergesConsumerName))
+		})
+		if err != nil {
+			return fmt.Errorf("subscribing to the forge's merges: %w", err)
+		}
+		if merges == nil {
+			return nil
+		}
+		defer func() { _ = merges.Unsubscribe() }()
+	}
+
 	// A connection that dropped is said, not discovered. A controller whose bus connection is gone
 	// is a mesh where nothing can be told anything.
 	gone := make(chan error, 1)

internal/link/receive_nats_test.go

@@ -590,3 +590,77 @@ func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) {
 type sameAgain struct{}
 
 func (sameAgain) Heard(context.Context, Report) (bool, error) { return false, nil }
+
+// manyTimesBusy is an upgrader whose plans are busy for its first merges, as another controller's hold is.
+type manyTimesBusy struct {
+	toldAbout
+	busy  int
+	calls int
+}
+
+func (u *manyTimesBusy) SourceMoved(context.Context, SourceMoved) error {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	u.calls++
+	if u.calls <= u.busy {
+		return errors.Join(ErrTryAgain, errors.New("another controller is working the plans"))
+	}
+	return nil
+}
+
+func (u *manyTimesBusy) called() int {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	return u.calls
+}
+
+// **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision
+// 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait
+// for a person; the delay grows, and is capped.
+func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) {
+	first, most := MergeHeldFirst, MergeHeldMost
+	MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond
+	t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most })
+	js := aBus(t)
+	up := &manyTimesBusy{busy: 8}
+	s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
+	if err := s.Follows(up); err != nil {
+		t.Fatal(err)
+	}
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go func() { _ = s.Serve(ctx) }()
+	eventually(t, "the merges consumer being made", func() bool {
+		_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
+		return err == nil
+	})
+
+	body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"})
+	if _, err := js.Context().Publish(MergedSubject, body); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 })
+	eventually(t, "the merge being acknowledged", func() bool {
+		info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
+		return err == nil && info.NumAckPending == 0 && info.NumPending == 0
+	})
+	if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 {
+		t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs)
+	}
+}
+
+// The delay a held merge waits grows from the first to the most and stays there.
+func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) {
+	var got []time.Duration
+	for _, held := range []time.Duration{0, 4 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
+		200 * time.Second, 20 * time.Minute} {
+		got = append(got, MergeRedeliverAfter(held))
+	}
+	want := []time.Duration{5 * time.Second, 5 * time.Second, 10 * time.Second, 25 * time.Second, 70 * time.Second,
+		200 * time.Second, 5 * time.Minute}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("held %d: waits %v; want %v", i, got[i], want[i])
+		}
+	}
+}

internal/link/serve.go

@@ -636,8 +636,19 @@ func short(commit string) string {
 	return commit
 }
 
-// mergeHeldFor is how long a merge the controller could not record yet waits before the bus hands it over again.
-const mergeHeldFor = 5 * time.Second
+// MergeHeldFirst is how long a merge the controller could not record yet waits before the bus hands it over
+// again, and MergeHeldMost the longest it ever waits: the delay grows with how long the merge has been held, from
+// the first to the most (novox/hq ADR 0407 decision 3). Variables so a test need not wait.
+var (
+	MergeHeldFirst = 5 * time.Second
+	MergeHeldMost  = 5 * time.Minute
+)
+
+// MergeRedeliverAfter is the delay for a merge held for the time given: that time, so the delay about doubles
+// with each hold, between MergeHeldFirst and MergeHeldMost.
+func MergeRedeliverAfter(heldFor time.Duration) time.Duration {
+	return min(max(heldFor, MergeHeldFirst), MergeHeldMost)
+}
 
 // sourceMoved acts on the forge's announcement of a merge. **A merge is never dropped** (issue 921, novox/hq ADR
 // 0407 decision 3): one the controller could not record for now (the plans held by another controller, the store
@@ -663,13 +674,14 @@ func (s *Server) sourceMoved(ctx context.Context, m Control) {
 		if ctx.Err() != nil {
 			return
 		}
-		if held := m.Hold(mergeHeldFor); held != nil {
+		delay := MergeRedeliverAfter(m.HeldFor())
+		if held := m.Hold(delay); held != nil {
 			s.log.Printf("%s/%s moved to %.8s and could not be held (%v): %v; left for the bus to hand over again",
 				moved.Owner, moved.Repo, moved.Commit, held, err)
 			return
 		}
 		s.log.Printf("%s/%s moved to %.8s and is held: %v; handed over again in %s", moved.Owner, moved.Repo,
-			moved.Commit, err, mergeHeldFor)
+			moved.Commit, err, delay)
 		return
 	}
 	if err != nil {