Remove an ask's original only when its sequence still holds it, and only with a worker (issue 334)

A seat's queue made again numbers from one, so an old dead letter's sequence
can name a live ask; deleting by number alone would drop it silently. An ask
with no worker would wait unseen while counted as delivered.

internal/link/deadletters_ask_test.go

@@ -100,3 +100,118 @@ func TestAnAskTheControllerDidNotMakeIsStillOnlyDropped(t *testing.T) {
 		}
 	}
 }
+
+// aBuildAskGivenUp publishes one build ask and lets the worker give it up.
+func aBuildAskGivenUp(t *testing.T, js *broker.JetStream, worker jetstream.Consumer, body string) DeadLetter {
+	t.Helper()
+	if _, err := js.Context().Publish(BuildWorkOf(TheBuildMachine), []byte(body)); err != nil {
+		t.Fatal(err)
+	}
+	return givenUp(t, js, worker)
+}
+
+// A copy that cannot be published: the original is already out of the queue, the dead letter is kept and
+// the answer says both; asked again once it can be published, it is delivered once.
+func TestAnAskWhoseCopyIsRefusedStaysKeptAndIsDeliveredOnTheNextTry(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	keeping(t, js)
+	worker := theBuildWorker(t, js)
+	d := aBuildAskGivenUp(t, js, worker, `{"module":"x"}`)
+
+	// The queue stops taking the ask's subject, so the copy's publish is refused by the server.
+	info, err := js.Context().StreamInfo(d.Stream)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := info.Config
+	taking := cfg.Subjects
+	cfg.Subjects = []string{"mesh.seat." + TheBuildMachine + ".accept.nothing"}
+	if _, err := js.Context().UpdateStream(&cfg); err != nil {
+		t.Fatal(err)
+	}
+	_, _, err = DeliverAgain(js.Context(), d.ID)
+	if err == nil || !strings.Contains(err.Error(), "still kept") || !strings.Contains(err.Error(), "was removed") {
+		t.Fatalf("a refused copy answered %v", err)
+	}
+	if _, err := js.Context().GetMsg(d.Stream, d.Sequence); !errors.Is(err, nats.ErrMsgNotFound) {
+		t.Fatalf("the original is still in the queue: %v", err)
+	}
+	if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
+		t.Fatalf("a refused copy let the dead letter go: %v", err)
+	}
+
+	cfg.Subjects = taking
+	if _, err := js.Context().UpdateStream(&cfg); err != nil {
+		t.Fatal(err)
+	}
+	retried, _, err := DeliverAgain(js.Context(), d.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(retried.Original, "no longer held") {
+		t.Fatalf("the retry said of the original: %q", retried.Original)
+	}
+	if again := next(t, worker); again == nil || string(again.Data()) != `{"module":"x"}` {
+		t.Fatal("the retry did not hand the ask to the worker")
+	} else {
+		_ = again.Ack()
+	}
+	if m := next(t, worker); m != nil {
+		t.Fatalf("handed twice: %s", m.Data())
+	}
+}
+
+// A queue made again numbers from one: the old dead letter's sequence then names a live ask, which is left
+// alone.
+func TestAnAskWhoseSequenceNamesAnotherMessageLeavesThatOneAlone(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	keeping(t, js)
+	d := aBuildAskGivenUp(t, js, theBuildWorker(t, js), `{"module":"old"}`)
+
+	// The build handover's way: the seat's queue deleted and made again, with its worker.
+	if err := js.Context().DeleteStream(d.Stream); err != nil {
+		t.Fatal(err)
+	}
+	if err := broker.RaiseSeats(js, []broker.DeclaredSeat{{Name: TheBuildMachine, Accepts: []string{"build"},
+		Emits: []string{"built"}}}, map[string]broker.Holder{TheBuildMachine: {Node: "anchor", Module: "builder"}}); err != nil {
+		t.Fatal(err)
+	}
+	live, err := js.Context().Publish(BuildWorkOf(TheBuildMachine), []byte(`{"module":"live"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if live.Sequence != d.Sequence {
+		t.Fatalf("the live ask is message %d, the dead letter names %d: the test does not set up the collision",
+			live.Sequence, d.Sequence)
+	}
+
+	delivered, _, err := DeliverAgain(js.Context(), d.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(delivered.Original, "another message") {
+		t.Fatalf("the answer said of the original: %q", delivered.Original)
+	}
+	if held, err := js.Context().GetMsg(d.Stream, d.Sequence); err != nil || string(held.Data) != `{"module":"live"}` {
+		t.Fatalf("the live ask at that sequence was touched: %v", err)
+	}
+}
+
+// No worker on the seat's queue: refused, kept, and nothing published.
+func TestAnAskIsNotDeliveredAgainWhileTheSeatHasNoWorker(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	keeping(t, js)
+	d := aBuildAskGivenUp(t, js, theBuildWorker(t, js), `{"module":"x"}`)
+	if err := js.Context().DeleteConsumer(d.Stream, d.Consumer); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "wait in") {
+		t.Fatalf("delivered with no worker: %v", err)
+	}
+	if _, err := js.Context().GetMsg(d.Stream, d.Sequence); err != nil {
+		t.Fatalf("a refusal removed the original: %v", err)
+	}
+	if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
+		t.Fatalf("a refusal let the dead letter go: %v", err)
+	}
+}