Keep what a consumer gives up on until a person delivers it again or drops it

Design 25 promised a dead-letter stream that did not exist: a message a
consumer gave up on stayed only in its source, which drops it after a week,
and its condition cleared when the advisories stopped (hq issue 330, ADR 0264).

internal/broker/derived.go

@@ -38,7 +38,8 @@ type Consumer struct {
 	Push bool
 	// AckWaitSeconds before an unacknowledged delivery is redelivered.
 	AckWaitSeconds int
-	// MaxDeliver before the message is dead-lettered; zero for the mesh's default.
+	// MaxDeliver is how often a message is handed over before the consumer gives it up; zero for no
+	// bound. What a consumer gives up is kept in DEAD_LETTERS by the controller (novox/hq issue 330).
 	MaxDeliver int
 	// MaxAckPending is how many deliveries the server lets stand unacknowledged at once; zero for
 	// the server's default, which is many. **One, for a consumer handled one at a time**
@@ -145,13 +146,16 @@ func ConsumerFor(p Principal) (Consumer, bool) {
 		return Consumer{}, false
 	}
 	sort.Strings(filters)
+	// And the events given up on and delivered again to this consumer alone (novox/hq issue 330).
+	filters = append(filters, AgainFilter(consumerDurable(p)))
 	return Consumer{
 		Name:           consumerDurable(p),
 		Stream:         consumerStream(p),
 		Filters:        filters,
 		AckWaitSeconds: 30,
 		MaxDeliver:     5,
-		Why:            "what " + p.Module + " declared it consumes; after max-deliver it dead-letters",
+		Why: "what " + p.Module + " declared it consumes; after max-deliver it gives an event up, and the " +
+			"controller keeps it in DEAD_LETTERS until a person delivers it again or drops it",
 	}, true
 }
 
@@ -234,7 +238,7 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool)
 //
 // **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied
 // it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot
-// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node
+// get a node to accept is not one to give up on, because the stream keeps only the newest per node
 // anyway — so there is exactly one message per node to redeliver, for as long as that node is away.
 func NodeConsumer(node string) Consumer {
 	return Consumer{

internal/broker/derived_test.go

@@ -61,8 +61,9 @@ func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) {
 	if !ok {
 		t.Fatal("a module that consumes got no consumer")
 	}
-	if len(c.Filters) != 2 {
-		t.Fatalf("expected both subjects as filters, got %v", c.Filters)
+	// Both, and its own share of what is delivered again (novox/hq issue 330).
+	if len(c.Filters) != 3 || c.Filters[2] != "mesh.again.one_audit.>" {
+		t.Fatalf("expected both subjects and its own again filter, got %v", c.Filters)
 	}
 	perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit",
 		Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})

internal/broker/jetstream.go

@@ -154,6 +154,15 @@ func (j *JetStream) EnsureStream(s Stream) error {
 		Description:       s.Why,
 	}
 	want.AllowDirect = s.Direct
+	if s.MaxBytes > 0 {
+		want.MaxBytes = s.MaxBytes
+	}
+	if s.DiscardNew {
+		want.Discard = nats.DiscardNew
+	}
+	if s.DuplicatesSeconds > 0 {
+		want.Duplicates = time.Duration(s.DuplicatesSeconds) * time.Second
+	}
 	if s.Retention == RetentionLastPerSubject {
 		// Last-per-subject is a limits stream with one message kept per subject, not a
 		// retention policy of its own — the state shape, spelled the way the server spells it.

internal/broker/nats.go

@@ -412,6 +412,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// both in the mesh's own account; the controller says each as a condition in the mesh's words.
 		// Named, not `$JS.EVENT.>`: the other advisories are every API call the mesh makes.
 		sub = append(sub, BusAdvisories...)
+		// **And what a consumer gave up on, kept and delivered again** (novox/hq issue 330): the notice
+		// acknowledged once the message is copied, the copy kept, and a message delivered again — an
+		// event to the one consumer that gave it up, an ask back onto its seat's queue, whose one worker
+		// is the consumer that gave it up.
+		pub = append(pub, "$JS.ACK."+DeadLetterNoticesStream+"."+ControllerName+".>", deadLetterPrefix+">",
+			againPrefix+">", "mesh.seat.*.accept.>")
 
 	case KindPerson:
 		// Tools, and nothing else. Every subject a person may publish is a tool call; a person

internal/broker/streams.go

@@ -3,11 +3,13 @@ package broker
 import (
 	"fmt"
 	"sort"
+	"strings"
 )
 
 // The mesh's own streams.
 //
-// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier
+// **These and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118; the two that keep what a
+// consumer gave up on added for issue 330). An earlier
 // reading had the controller create *every* stream at genesis, from a fixed set. That is only the
 // mesh's own half: a seat's streams are created when the module declaring it is registered, and a
 // module's durable consumers when it is assigned — neither of which has happened at genesis. What
@@ -50,11 +52,80 @@ type Stream struct {
 	// Direct lets a client read a subject's last message without a consumer, which is how a
 	// runtime reads its own membership with no JetStream API beyond one request (ADR 0160).
 	Direct bool
+	// MaxBytes bounds the stream's size, zero for unbounded. With DiscardNew a full stream refuses
+	// what comes next rather than dropping what it holds: the publisher is told, and says so.
+	MaxBytes   int64
+	DiscardNew bool
+	// DuplicatesSeconds is the window in which a message id published twice is kept once; zero for the
+	// server's default (two minutes).
+	DuplicatesSeconds int
 }
 
 // AssignmentsStream holds every assignment's membership, the newest per subject.
 const AssignmentsStream = "ASSIGNMENTS"
 
+// What a durable consumer gave up on is kept (novox/hq issue 330, design 25 §3).
+//
+// **The server says it and keeps it; the controller copies it.** A consumer that handed a message over
+// as often as it may stops offering it and publishes `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES` with
+// the stream and the message's sequence. DeadLetterNoticesStream captures those advisories as the
+// server publishes them, so one said while no controller listens is still there when one starts. The
+// controller's consumer on it fetches the given-up message by its sequence, while the source stream
+// still holds it, and keeps a copy in DeadLettersStream under DeadLetterSubject, with the consumer,
+// the subject, how often it was handed over and when it was given up. It stays there until a person
+// delivers it again or drops it, with why; a condition is open for as long as it does.
+const (
+	DeadLetterNoticesStream = "DEAD_LETTER_NOTICES"
+	DeadLettersStream       = "DEAD_LETTERS"
+	// MaxDeliveriesAdvisories is the subject the server says a given-up message on.
+	MaxDeliveriesAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>"
+	deadLetterPrefix        = "mesh.events.dead."
+	againPrefix             = "mesh.again."
+	// DeadLettersBytes bounds the kept copies. Full, the stream refuses the next copy, which is said
+	// as the consumer's condition; it never drops one it holds.
+	DeadLettersBytes = 256 << 20
+)
+
+// DeadLetterSubject is where a message one consumer gave up on is kept: `mesh.events.dead.<stream>.<consumer>`.
+func DeadLetterSubject(stream, consumer string) string {
+	return deadLetterPrefix + stream + "." + consumer
+}
+
+// DeadLetterOf is the stream and consumer a kept message's subject names; false for any other subject.
+func DeadLetterOf(subject string) (stream, consumer string, ok bool) {
+	rest, found := strings.CutPrefix(subject, deadLetterPrefix)
+	if !found {
+		return "", "", false
+	}
+	stream, consumer, ok = strings.Cut(rest, ".")
+	return stream, consumer, ok && stream != "" && consumer != "" && !strings.Contains(consumer, ".")
+}
+
+// AgainSubject is where an event given up on is delivered again to the one consumer that gave it up,
+// and to nobody else: `mesh.again.<consumer>.` and the original subject without its `mesh.`. Every
+// consumer on EVENTS filters its own (AgainFilter), and a module's runtime reads the event's key from
+// the tokens around `.event.`, so the handler sees the same key it saw the first time.
+func AgainSubject(consumer, original string) string {
+	return againPrefix + consumer + "." + strings.TrimPrefix(original, "mesh.")
+}
+
+// AgainFilter is the one consumer's share of the subjects events are delivered again on.
+func AgainFilter(consumer string) string { return againPrefix + consumer + ".>" }
+
+// OriginalOfAgain is the subject an event delivered again was first published on; false for a subject
+// that is not one delivered again.
+func OriginalOfAgain(subject string) (string, bool) {
+	rest, found := strings.CutPrefix(subject, againPrefix)
+	if !found {
+		return "", false
+	}
+	_, original, ok := strings.Cut(rest, ".")
+	if !ok || original == "" {
+		return "", false
+	}
+	return "mesh." + original, true
+}
+
 // MeshStreams is the foundation set, in the order a person reads it.
 //
 // **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
@@ -97,7 +168,9 @@ func MeshStreams() []Stream {
 			// A seat's own events ride here too: they are 1:many like any event, and the
 			// `event` token keeps them clear of both the seat's work queue (`accept`) and its
 			// tools (`tool`), which must not be persisted.
-			Subjects:          []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"},
+			// And an event given up on, delivered again to the one consumer that gave it up
+			// (novox/hq issue 330): under `mesh.again.<consumer>.`, which only that consumer filters.
+			Subjects:          []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", againPrefix + ">"},
 			Retention:         RetentionLimits,
 			MaxAge:            7 * 24 * 60 * 60,
 			MaxMsgsPerSubject: 10000,
@@ -105,6 +178,26 @@ func MeshStreams() []Stream {
 				"excluded by the event token; per-subject caps keep a noisy emitter from " +
 				"evicting a quiet one without splitting the stream",
 		},
+		{
+			Name:      DeadLetterNoticesStream,
+			Subjects:  []string{MaxDeliveriesAdvisories},
+			Retention: RetentionWorkQueue,
+			MaxAge:    7 * 24 * 60 * 60,
+			Why: "the server's word that a consumer gave up on a message, kept until the controller has " +
+				"copied the message into DEAD_LETTERS (novox/hq issue 330); a week, the longest the source " +
+				"streams keep what they are about",
+		},
+		{
+			Name:              DeadLettersStream,
+			Subjects:          []string{deadLetterPrefix + ">"},
+			Retention:         RetentionLimits,
+			MaxBytes:          DeadLettersBytes,
+			DiscardNew:        true,
+			DuplicatesSeconds: 24 * 60 * 60,
+			Why: "every message a consumer gave up on, with its consumer, subject, deliveries and when, kept " +
+				"until a person delivers it again or drops it with why (novox/hq issue 330); no age, and full " +
+				"it refuses the next copy rather than drop one it holds",
+		},
 	}
 }
 
@@ -298,7 +391,7 @@ const EventsStream = "EVENTS"
 //
 // **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
 // not the server's (window.go): a message is held with a nak-and-delay until the controller either
-// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being
+// takes it or gives up and says so. A max-deliver here would give up on a push that was being
 // held through a store restart — the exact message the stream exists to protect — some minutes
 // before the controller had finished deciding about it.
 func MeshConsumers() []Consumer {
@@ -314,7 +407,7 @@ func MeshConsumers() []Consumer {
 		{
 			Name:           ControllerName,
 			Stream:         "EVENTS",
-			Filters:        ControllerFollows,
+			Filters:        append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName)),
 			Push:           true,
 			AckWaitSeconds: 30,
 			MaxDeliver:     5,
@@ -331,8 +424,18 @@ func MeshConsumers() []Consumer {
 			Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " +
 				"from the build records (issue 214), a provider's failing word said again (ADR 0224)",
 			Why: "the events the mesh's own controller reacts to, one at a time; after " +
-				"max-deliver it dead-letters, because an announcement it cannot act on will not " +
-				"become actionable",
+				"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
+				"a person delivers it again or drops it",
+		},
+		// What the server said a consumer gave up on (novox/hq issue 330): copied into DEAD_LETTERS and
+		// acknowledged. No max-deliver: a notice the controller could not copy is offered again, and said.
+		{
+			Name:           ControllerName,
+			Stream:         DeadLetterNoticesStream,
+			Push:           true,
+			AckWaitSeconds: 30,
+			Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
+				"because a notice it gave up on would lose the message it is about",
 		},
 	}
 }

internal/broker/streams_test.go

@@ -127,6 +127,10 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
 		"NODES":       RetentionLastPerSubject,
 		"EVENTS":      RetentionLimits,
 		"ASSIGNMENTS": RetentionLastPerSubject,
+		// What a consumer gave up on (novox/hq issue 330): the server's notice taken once, the message
+		// kept until somebody acts.
+		"DEAD_LETTER_NOTICES": RetentionWorkQueue,
+		"DEAD_LETTERS":        RetentionLimits,
 	}
 	got := map[string]Retention{}
 	for _, s := range MeshStreams() {

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.*.accept.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/broker/writers.go

@@ -133,6 +133,11 @@ var WritersTable = []WriterRow{
 		Others: "—"},
 	{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
 		Others: "the build seat reads"},
+	// What a consumer gave up on (novox/hq issue 330): copied by the controller from the server's notice,
+	// and delivered again or dropped only through its verb, with why.
+	{State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream,
+		Others:   "read, delivered again or dropped through the controller's dead-letters verb",
+		Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: isController},
 }
 
 // CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives

internal/broker/writers_test.go

@@ -30,6 +30,7 @@ var designRows = []string{
 	"a provider's standing",
 	"the operator-channel's open messages",
 	"the facts snapshot",
+	"a message a consumer gave up on",
 }
 
 func TestTheWritersTableIsTheDesigns(t *testing.T) {