Remove only a move's proven second copy, and never delete a handed-out original (hq issue 558)

Second review of #276: a dead letter delivered again beside its original
shares its request id and was removed as a copy, losing the ask; and a
publish that outlived the pause could delete an original a holder had
taken. A copy now names the message it copies, removals are said, the
publish waits inside the pause's margin, and an original handed out
meanwhile keeps its place while the copy goes.

internal/link/queue_order_test.go

@@ -200,9 +200,9 @@ func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
 	check := aCheck(now.Add(-2*time.Minute), "/c")
 	publishAsk(t, js, check)
 	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
-	publishAsk(t, js, check) // the copy, published; the original never deleted
-	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
-		t.Fatal(err)
+	moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
+		t.Fatalf("removed %v: %v", pass.Removed, err)
 	}
 	var got []string
 	for _, a := range queued(t, js) {
@@ -234,7 +234,7 @@ func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
 		}
 		_ = msgs[0].Ack()
 	}
-	publishAsk(t, js, taken)
+	moveCopyOf(t, js, taken.ID)
 	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
 		t.Fatal(err)
 	}
@@ -249,6 +249,83 @@ func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
 	}
 }
 
+// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
+// would leave it.
+func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
+	t.Helper()
+	for _, a := range queued(t, js) {
+		if a.ID != id {
+			continue
+		}
+		body, _ := json.Marshal(a.Request)
+		msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
+		msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
+		msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
+		if _, err := js.Context().PublishMsg(msg); err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+	t.Fatalf("%s is not queued", id)
+}
+
+// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
+// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
+// copy, whose headers the delivery carries.
+func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	ask := aCheck(time.Now(), "/again")
+	publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
+	publishAsk(t, js, ask)
+	publishAsk(t, js, aDelivery(time.Now(), "/d"))
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
+		t.Fatal(err)
+	}
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	var original *nats.Msg
+	for original == nil {
+		msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
+		if err != nil || len(msgs) == 0 {
+			t.Fatalf("nothing to take: %v", err)
+		}
+		var r BuildRequest
+		_ = json.Unmarshal(msgs[0].Data, &r)
+		if r.ID == ask.ID {
+			original = msgs[0] // given up on, never settled: dead, and kept
+			continue
+		}
+		_ = msgs[0].Ack()
+	}
+	// Delivered again as DeliverAgain does: its headers, and its own.
+	again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
+	for k, v := range original.Header {
+		again.Header[k] = v
+	}
+	again.Header.Set(AgainHeader, "7")
+	if _, err := js.Context().PublishMsg(again); err != nil {
+		t.Fatal(err)
+	}
+	pass, err := PutInOrder(ctx, js, TheBuildMachine)
+	if err != nil || len(pass.Removed) != 0 {
+		t.Fatalf("removed %v: %v", pass.Removed, err)
+	}
+	n := 0
+	for _, a := range queued(t, js) {
+		if a.ID == ask.ID {
+			n++
+		}
+	}
+	if n != 2 {
+		t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
+	}
+}
+
 // Two passes at once: nothing is built twice, and nothing is lost.
 func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
 	js := aBusWithTheBuildRole(t)
@@ -327,8 +404,8 @@ func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
 	publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
 	read := queued(t, js)[0]
 	publishAsk(t, js, aDelivery(now, "/d"))
-	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
-		t.Fatalf("moved %v: %v", moved, err)
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
+		t.Fatalf("moved %v: %v", pass.Moved, err)
 	}
 	removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
 	if err != nil || removed == 0 || removed == read.Seq {
@@ -354,8 +431,8 @@ func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
 	first := queued(t, js)[0].Stored
 	time.Sleep(50 * time.Millisecond)
 	publishAsk(t, js, aDelivery(time.Now(), "/d"))
-	if moved, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(moved) != 1 {
-		t.Fatalf("moved %v: %v", moved, err)
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
+		t.Fatalf("moved %v: %v", pass.Moved, err)
 	}
 	asks := queued(t, js)
 	if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {