Give a work-queue stream its one consumer, drop-only dead letters (issue 1178)

internal/link/deadletters.go

@@ -45,6 +45,10 @@ const (
 	// DeadHeaderAgesOut when its stream's maximum age runs out, counted from when it was first published.
 	DeadHeaderRestricted = "Mesh-Dead-Restricted"
 	DeadHeaderAgesOut    = "Mesh-Dead-Ages-Out"
+	// DeadHeaderWorkQueue says the message came from a work-queue stream (novox/hq ADR 0436 §6): aged and
+	// shown as a restricted stream's are (it sets DeadHeaderRestricted too), but never delivered again —
+	// only dropped.
+	DeadHeaderWorkQueue = "Mesh-Dead-Work-Queue"
 	// AgainHeader marks a message delivered again, with the kept copy's id it came from.
 	AgainHeader = "Mesh-Delivered-Again"
 )
@@ -74,6 +78,10 @@ type DeadLetter struct {
 	Restricted bool      `json:"restricted,omitempty"`
 	AgesOut    time.Time `json:"ages_out,omitzero"`
 	Withheld   string    `json:"withheld,omitempty"`
+	// WorkQueue says it came from a work-queue stream (novox/hq ADR 0436 §6): shown and aged as a restricted
+	// stream's are (Restricted is also set), but AgainTo refuses it unconditionally — only dropped, never
+	// delivered again.
+	WorkQueue bool `json:"work-queue,omitempty"`
 	// Body and Headers are the message itself, given only for one dead letter asked by its id, and never for
 	// a restricted one; a header with several values keeps them all.
 	Body    string              `json:"body,omitempty"`
@@ -91,17 +99,19 @@ const leastLeftToDeliver = time.Minute
 const withheldWords = "its body and its own headers are not shown: it came from a restricted stream, which only " +
 	"the modules its grant names may read (novox/hq ADR 0376)"
 
-// sourceAge is whether a stream is restricted (novox/hq ADR 0363 §4), and its maximum age: read from the stream
-// itself, where the controller says both. A stream that no longer exists is neither.
-func sourceAge(js nats.JetStreamContext, stream string) (bool, time.Duration, error) {
+// sourceAge is whether a stream is restricted (novox/hq ADR 0363 §4; a work-queue stream is restricted too,
+// novox/hq ADR 0436 §6, aged and hidden the same way), whether it is a work queue, and its maximum age: read
+// from the stream itself, where the controller says all three. A stream that no longer exists is neither.
+func sourceAge(js nats.JetStreamContext, stream string) (restricted, workQueue bool, maxAge time.Duration, err error) {
 	info, err := js.StreamInfo(stream)
 	if errors.Is(err, nats.ErrStreamNotFound) {
-		return false, 0, nil
+		return false, false, 0, nil
 	}
 	if err != nil {
-		return false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err)
+		return false, false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err)
 	}
-	return info.Config.Metadata[broker.RestrictedMetadata] == "true", info.Config.MaxAge, nil
+	workQueue = info.Config.Retention == nats.WorkQueuePolicy
+	return info.Config.Metadata[broker.RestrictedMetadata] == "true", workQueue, info.Config.MaxAge, nil
 }
 
 // ttlOf is a time left as a message's time to live, in whole seconds and never more than what is left.
@@ -135,13 +145,16 @@ func KeepDeadLetter(js nats.JetStreamContext, notice []byte) (DeadLetter, error)
 	// **A restricted stream's message ages out with its stream** (novox/hq ADR 0376 §3): kept with a time to live
 	// of what is left of the stream's maximum age, counted from when it was first published; one already past it
 	// is kept as a record without its body. Every other dead letter keeps no age, as before (ADR 0264).
-	restricted, maxAge, err := sourceAge(js, a.Stream)
+	restricted, workQueue, maxAge, err := sourceAge(js, a.Stream)
 	if err != nil {
 		return DeadLetter{}, err
 	}
 	if restricted {
 		kept.Header.Set(DeadHeaderRestricted, "true")
 	}
+	if workQueue {
+		kept.Header.Set(DeadHeaderWorkQueue, "true")
+	}
 	raw, err := js.GetMsg(a.Stream, a.StreamSeq)
 	agesOut := time.Time{}
 	if err == nil && restricted && maxAge > 0 {
@@ -205,6 +218,7 @@ func deadLetterOf(id uint64, subject string, h nats.Header, data []byte, whole b
 	d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderGaveUp))
 	d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderPublished))
 	d.Restricted = h.Get(DeadHeaderRestricted) == "true"
+	d.WorkQueue = h.Get(DeadHeaderWorkQueue) == "true"
 	d.AgesOut, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderAgesOut))
 	if whole {
 		d.Body = string(data)
@@ -349,6 +363,13 @@ func AgainTo(js nats.JetStreamContext, d DeadLetter) (string, error) {
 	case d.Subject == "":
 		return "", fmt.Errorf("dead letter %d does not say the subject it was published on, so it cannot be "+
 			"delivered again. Drop it", d.ID)
+	case d.WorkQueue:
+		// A work-queue stream's one consumer takes its messages in order (novox/hq ADR 0436 clause 6):
+		// delivering a given-up one back in would let it jump ahead of whatever the stream already holds, so
+		// unlike a module's ordinary stream (ADR 0363 §7) it is never delivered again, however much of its
+		// age is left.
+		return "", fmt.Errorf("dead letter %d is from %s, a work queue: it is only dropped, never delivered "+
+			"again (novox/hq ADR 0436 clause 6). Drop it", d.ID, d.Stream)
 	case broker.TheControllersAsk(d.Stream, d.Subject):
 		// The seat's worker takes it, and only while it is on the bus: without one the queue would keep
 		// the ask for a holder that may never come, and it would be let go as delivered meanwhile.
@@ -593,7 +614,7 @@ func AgingDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string
 		stream, consumer, _ := strings.Cut(key, ".")
 		is, known := restricted[stream]
 		if !known {
-			r, _, err := sourceAge(js, stream)
+			r, _, _, err := sourceAge(js, stream)
 			if err != nil {
 				return nil, err
 			}

internal/link/deadletters_work_queue_test.go

@@ -0,0 +1,50 @@
+package link
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// A work-queue stream's dead letters are kept and shown as a restricted stream's are — aged with the
+// stream, never with their body or headers — and can only be dropped, never delivered again (novox/hq ADR
+// 0436 §6, changing ADR 0363 §7 for such a stream).
+
+// 1178/AC3: a dead letter's headers say it came from a work-queue stream, and reading it back carries that
+// (and the restricted flag its aging and hiding already key off, deadLetterOf being the one place both are
+// read: see DeadLetter.Restricted's doc).
+func TestAWorkQueueDeadLettersHeaderIsReadBack(t *testing.T) {
+	agesOut := time.Now().Add(time.Hour).UTC()
+	h := nats.Header{}
+	h.Set(DeadHeaderStream, "MAIL_SENDS")
+	h.Set(DeadHeaderConsumer, "novox_mesh-mail_send")
+	h.Set(DeadHeaderSubject, "mesh.stream.mesh-mail.event.mail.send.approved")
+	h.Set(DeadHeaderRestricted, "true")
+	h.Set(DeadHeaderWorkQueue, "true")
+	h.Set(DeadHeaderAgesOut, agesOut.Format(time.RFC3339Nano))
+
+	d := deadLetterOf(42, "mesh.stream.mesh-mail.event.mail.send.approved", h, []byte("secret body"), false)
+	if !d.WorkQueue {
+		t.Fatalf("a work-queue dead letter read as %+v", d)
+	}
+	if !d.Restricted {
+		t.Fatalf("a work-queue dead letter is not read as restricted, so it would be shown with its body: %+v", d)
+	}
+	if !d.AgesOut.Equal(agesOut) {
+		t.Fatalf("ages out at %v, want %v — it should age with its stream", d.AgesOut, agesOut)
+	}
+}
+
+// 1178/AC3: asking to deliver a work-queue dead letter again is refused, naming ADR 0436 clause 6 — before
+// anything is asked of the bus, so it holds whatever max-deliver or age is left.
+func TestAWorkQueueDeadLetterIsNeverDeliveredAgain(t *testing.T) {
+	d := DeadLetter{ID: 7, Stream: "MAIL_SENDS", Consumer: "novox_mesh-mail_send",
+		Subject: "mesh.stream.mesh-mail.event.mail.send.approved", WorkQueue: true, Restricted: true,
+		AgesOut: time.Now().Add(time.Hour)}
+	if _, err := AgainTo(nil, d); err == nil || !strings.Contains(err.Error(), "ADR 0436 clause 6") ||
+		!strings.Contains(err.Error(), "Drop it") {
+		t.Fatalf("delivering a work-queue dead letter again: %v; want a refusal naming ADR 0436 clause 6", err)
+	}
+}