Age a restricted stream's dead letters out with it and never show their body (issue 728)

DEAD_LETTERS kept a message given up on MAIL forever and showed it whole to any caller of dead-letters, outside the grant and past MAIL's week. Per ADR 0376 §3-§6: the kept copy carries a per-message time to live of what is left of its stream's age (a copy already past it is kept without its body), DEAD_LETTERS and EVENTS allow per-message TTL, the list and show give no body or own headers for it, delivering it again needs a minute left and the copy keeps the time left, and its max-deliveries condition clears saying how many aged out undelivered and from which stream.

internal/broker/jetstream.go

@@ -173,6 +173,10 @@ func (j *JetStream) EnsureStream(s Stream) error {
 		Description:       s.Why,
 	}
 	want.AllowDirect = s.Direct
+	want.AllowMsgTTL = s.AllowMsgTTL
+	if s.Restricted {
+		want.Metadata = map[string]string{RestrictedMetadata: "true"}
+	}
 	if s.MaxBytes > 0 {
 		want.MaxBytes = s.MaxBytes
 	}

internal/broker/modstreams.go

@@ -77,6 +77,8 @@ func (s ModuleStream) Stream() Stream {
 		MaxAge:    s.MaxAgeDays * 24 * 60 * 60,
 		MaxBytes:  int64(s.MaxMiB) * 1024 * 1024,
 		Why:       why,
+		// Said on the bus, so a dead letter kept from it ages out with it (novox/hq ADR 0376 §3).
+		Restricted: s.Restricted,
 	}
 }
 

internal/broker/msgttl_live_test.go

@@ -0,0 +1,86 @@
+package broker
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// **DEAD_LETTERS and EVENTS take a message's own time to live once updated in place** (novox/hq ADR 0376 §3, §5),
+// on the pinned server: each made as it was before, without one, is brought to its definition by the controller's
+// assertion, keeps what it held, and removes a message once its time to live passes. A restricted module stream says
+// so in its metadata; one without a grant does not.
+//
+//	go test ./internal/broker/ -run TestTheStreamsTakeAMessagesTimeToLive
+func TestTheStreamsTakeAMessagesTimeToLive(t *testing.T) {
+	js, err := Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer js.Close()
+	for _, s := range MeshStreams() {
+		if s.Name != EventsStream && s.Name != DeadLettersStream {
+			if s.AllowMsgTTL {
+				t.Errorf("%s takes a time to live of a message's own, which nothing asks of it", s.Name)
+			}
+			continue
+		}
+		if !s.AllowMsgTTL {
+			t.Fatalf("%s is not defined to take a message's time to live", s.Name)
+		}
+		_ = js.js.DeleteStream(s.Name)
+		before := s
+		before.AllowMsgTTL = false
+		if err := js.EnsureStream(before); err != nil {
+			t.Fatalf("%s as it was: %v", s.Name, err)
+		}
+		subject := DeadLetterSubject("EVENTS", "ace_reader")
+		if s.Name == EventsStream {
+			subject = "mesh.mod.gitea.event.pull.merged"
+		}
+		if _, err := js.js.Publish(subject, []byte("held")); err != nil {
+			t.Fatal(err)
+		}
+		if err := js.EnsureStream(s); err != nil {
+			t.Fatalf("%s updated in place: %v", s.Name, err)
+		}
+		info, err := js.js.StreamInfo(s.Name)
+		if err != nil || !info.Config.AllowMsgTTL || info.State.Msgs != 1 {
+			t.Fatalf("%s after the update: %+v (%v)", s.Name, info, err)
+		}
+		msg := &nats.Msg{Subject: subject, Data: []byte("brief"), Header: nats.Header{}}
+		msg.Header.Set(nats.MsgTTLHdr, "1s")
+		ack, err := js.js.PublishMsg(msg)
+		if err != nil {
+			t.Fatalf("%s refused a message's time to live: %v", s.Name, err)
+		}
+		deadline := time.Now().Add(8 * time.Second)
+		for {
+			if _, err := js.js.GetMsg(s.Name, ack.Sequence); err != nil {
+				break
+			}
+			if time.Now().After(deadline) {
+				t.Fatalf("%s still holds a message past its time to live", s.Name)
+			}
+			time.Sleep(200 * time.Millisecond)
+		}
+		if info, _ := js.js.StreamInfo(s.Name); info.State.Msgs != 1 {
+			t.Fatalf("%s holds %d, want only the message without an age", s.Name, info.State.Msgs)
+		}
+		_ = js.js.PurgeStream(s.Name)
+	}
+	if !mailStream.Stream().Restricted || newsStream.Stream().Restricted {
+		t.Fatal("a module stream does not say whether it is restricted")
+	}
+	_ = js.js.DeleteStream("MAIL")
+	if err := js.EnsureStream(mailStream.Stream()); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.js.DeleteStream("MAIL") }()
+	if info, _ := js.js.StreamInfo("MAIL"); info.Config.Metadata[RestrictedMetadata] != "true" {
+		t.Fatalf("MAIL's metadata: %v", info.Config.Metadata)
+	}
+}

internal/broker/streams.go

@@ -59,8 +59,17 @@ type Stream struct {
 	// DuplicatesSeconds is the window in which a message id published twice is kept once; zero for the
 	// server's default (two minutes).
 	DuplicatesSeconds int
+	// AllowMsgTTL lets a message carry a time to live of its own (`Nats-TTL`), for what must not outlive the
+	// stream it came from (novox/hq ADR 0376 §3, §5).
+	AllowMsgTTL bool
+	// Restricted marks a module's stream restricted by a grant (novox/hq ADR 0363 §4), in the stream's own
+	// metadata, so whoever keeps one of its messages reads it from the bus (ADR 0376 §3).
+	Restricted bool
 }
 
+// RestrictedMetadata is the key a restricted stream's metadata says it under.
+const RestrictedMetadata = "mesh-restricted"
+
 // AssignmentsStream holds every assignment's membership, the newest per subject.
 const AssignmentsStream = "ASSIGNMENTS"
 
@@ -174,6 +183,8 @@ func MeshStreams() []Stream {
 			Retention:         RetentionLimits,
 			MaxAge:            7 * 24 * 60 * 60,
 			MaxMsgsPerSubject: 10000,
+			// A restricted dead letter delivered again keeps what is left of its stream's age (ADR 0376 §5).
+			AllowMsgTTL: true,
 			Why: "a subscriber that was down catches up; tool traffic under the same prefix is " +
 				"excluded by the event token; per-subject caps keep a noisy emitter from " +
 				"evicting a quiet one without splitting the stream",
@@ -194,9 +205,11 @@ func MeshStreams() []Stream {
 			MaxBytes:          DeadLettersBytes,
 			DiscardNew:        true,
 			DuplicatesSeconds: 24 * 60 * 60,
+			AllowMsgTTL:       true,
 			Why: "every message a consumer gave up on, with its consumer, subject, deliveries and when, kept " +
-				"until a person delivers it again or drops it with why (novox/hq issue 330); no age, and full " +
-				"it refuses the next copy rather than drop one it holds",
+				"until a person delivers it again or drops it with why (novox/hq issue 330); no age but a restricted " +
+				"stream's message, which ages out with its stream (ADR 0376), and full it refuses the next copy " +
+				"rather than drop one it holds",
 		},
 	}
 }

internal/link/deadletters.go

@@ -41,6 +41,10 @@ const (
 	DeadHeaderLost       = "Mesh-Dead-Lost"
 	// DeadHeaderMsgID is the message's own de-duplication id, kept, since the copy carries its own.
 	DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
+	// DeadHeaderRestricted says the message came from a restricted stream (novox/hq ADR 0376), and
+	// DeadHeaderAgesOut when its stream's maximum age runs out, counted from when it was first published.
+	DeadHeaderRestricted = "Mesh-Dead-Restricted"
+	DeadHeaderAgesOut    = "Mesh-Dead-Ages-Out"
 	// AgainHeader marks a message delivered again, with the kept copy's id it came from.
 	AgainHeader = "Mesh-Delivered-Again"
 )
@@ -65,12 +69,46 @@ type DeadLetter struct {
 	// Original says what became of the ask it was kept from, in its seat's work queue, when it was
 	// delivered again (novox/hq issue 334).
 	Original string `json:"original,omitempty"`
-	// Body and Headers are the message itself, given only for one dead letter asked by its id; a header
-	// with several values keeps them all.
+	// Restricted says it came from a restricted stream (novox/hq ADR 0376): its body and its own headers are
+	// never shown, said in Withheld, and it ages out at AgesOut, when its stream's maximum age runs out.
+	Restricted bool      `json:"restricted,omitempty"`
+	AgesOut    time.Time `json:"ages_out,omitzero"`
+	Withheld   string    `json:"withheld,omitempty"`
+	// Body and Headers are the message itself, given only for one dead letter asked by its id, and never for
+	// a restricted one; a header with several values keeps them all.
 	Body    string              `json:"body,omitempty"`
 	Headers map[string][]string `json:"headers,omitempty"`
 }
 
+// deadLetterNow is the clock dead letters age by: a variable, so a test can move it.
+var deadLetterNow = time.Now
+
+// leastLeftToDeliver is the least of its stream's age a restricted dead letter must have left to be delivered
+// again (novox/hq ADR 0376 §5).
+const leastLeftToDeliver = time.Minute
+
+// withheldWords is what the dead-letters verb says in place of a restricted dead letter's body.
+const withheldWords = "its body and its own headers are not shown: it came from a restricted stream, which only " +
+	"the modules its grant names may read (novox/hq ADR 0376)"
+
+// sourceAge is whether a stream is restricted (novox/hq ADR 0363 §4), and its maximum age: read from the stream
+// itself, where the controller says both. A stream that no longer exists is neither.
+func sourceAge(js nats.JetStreamContext, stream string) (bool, time.Duration, error) {
+	info, err := js.StreamInfo(stream)
+	if errors.Is(err, nats.ErrStreamNotFound) {
+		return false, 0, nil
+	}
+	if err != nil {
+		return false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err)
+	}
+	return info.Config.Metadata[broker.RestrictedMetadata] == "true", info.Config.MaxAge, nil
+}
+
+// ttlOf is a time left as a message's time to live, in whole seconds and never more than what is left.
+func ttlOf(left time.Duration) string {
+	return strconv.FormatInt(int64(left/time.Second), 10) + "s"
+}
+
 // maxDeliveries is the part of the server's notice a kept message is made from.
 type maxDeliveries struct {
 	Stream     string    `json:"stream"`
@@ -94,8 +132,28 @@ func KeepDeadLetter(js nats.JetStreamContext, notice []byte) (DeadLetter, error)
 		gaveUp = time.Now()
 	}
 	kept := &nats.Msg{Subject: broker.DeadLetterSubject(a.Stream, a.Consumer), Header: nats.Header{}}
+	// **A restricted stream's message ages out with its stream** (novox/hq ADR 0376 §3): kept with a time to live
+	// of what is left of the stream's maximum age, counted from when it was first published; one already past it
+	// is kept as a record without its body. Every other dead letter keeps no age, as before (ADR 0264).
+	restricted, maxAge, err := sourceAge(js, a.Stream)
+	if err != nil {
+		return DeadLetter{}, err
+	}
+	if restricted {
+		kept.Header.Set(DeadHeaderRestricted, "true")
+	}
 	raw, err := js.GetMsg(a.Stream, a.StreamSeq)
+	agesOut := time.Time{}
+	if err == nil && restricted && maxAge > 0 {
+		agesOut = raw.Time.Add(maxAge)
+		kept.Header.Set(DeadHeaderAgesOut, agesOut.UTC().Format(time.RFC3339Nano))
+	}
 	switch {
+	case err == nil && !agesOut.IsZero() && agesOut.Sub(deadLetterNow()) < time.Second:
+		kept.Header.Set(DeadHeaderSubject, raw.Subject)
+		kept.Header.Set(DeadHeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano))
+		kept.Header.Set(DeadHeaderLost, fmt.Sprintf("message %d of %s had passed its stream's maximum age of %s when "+
+			"the notice was taken, so it is kept without its body (novox/hq ADR 0376)", a.StreamSeq, a.Stream, maxAge))
 	case err == nil:
 		for k, v := range raw.Header {
 			if strings.HasPrefix(k, "Nats-") {
@@ -109,6 +167,9 @@ func KeepDeadLetter(js nats.JetStreamContext, notice []byte) (DeadLetter, error)
 		kept.Header.Set(DeadHeaderSubject, raw.Subject)
 		kept.Header.Set(DeadHeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano))
 		kept.Data = raw.Data
+		if !agesOut.IsZero() {
+			kept.Header.Set(nats.MsgTTLHdr, ttlOf(agesOut.Sub(deadLetterNow())))
+		}
 	case errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, nats.ErrStreamNotFound):
 		kept.Header.Set(DeadHeaderLost, fmt.Sprintf("%s no longer held message %d when the notice was taken: %v",
 			a.Stream, a.StreamSeq, err))
@@ -143,6 +204,8 @@ func deadLetterOf(id uint64, subject string, h nats.Header, data []byte, whole b
 	d.Deliveries, _ = strconv.ParseUint(h.Get(DeadHeaderDeliveries), 10, 64)
 	d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderGaveUp))
 	d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderPublished))
+	d.Restricted = h.Get(DeadHeaderRestricted) == "true"
+	d.AgesOut, _ = time.Parse(time.RFC3339Nano, h.Get(DeadHeaderAgesOut))
 	if whole {
 		d.Body = string(data)
 		d.Headers = map[string][]string{}
@@ -230,6 +293,9 @@ func DeadLetters(js nats.JetStreamContext, consumer string, most int) ([]DeadLet
 		if consumer != "" && consumer != d.Consumer && consumer != d.Stream+"."+d.Consumer {
 			continue
 		}
+		if d.Restricted {
+			d.Withheld = withheldWords
+		}
 		out = append(out, d)
 	}
 	return out, total, nil
@@ -238,8 +304,21 @@ func DeadLetters(js nats.JetStreamContext, consumer string, most int) ([]DeadLet
 // ErrNoDeadLetter is a dead letter's id DEAD_LETTERS does not hold.
 var ErrNoDeadLetter = errors.New("no such dead letter")
 
-// DeadLetterNamed is one kept message whole: what it said, and the headers it said it with.
+// DeadLetterNamed is one kept message whole: what it said, and the headers it said it with — **but for one from
+// a restricted stream, whose body and own headers are never shown** (novox/hq ADR 0376 §4). A grant names modules,
+// and this is asked by people and by tool runners' bus accounts, which a grant never names: withholding the body
+// from every caller is that grant's check, said plainly.
 func DeadLetterNamed(js nats.JetStreamContext, id uint64) (DeadLetter, error) {
+	d, err := deadLetterWhole(js, id)
+	if err != nil || !d.Restricted {
+		return d, err
+	}
+	d.Body, d.Headers, d.Withheld = "", nil, withheldWords
+	return d, nil
+}
+
+// deadLetterWhole is one kept message whole, a restricted one's body too: for delivering it again, never to show.
+func deadLetterWhole(js nats.JetStreamContext, id uint64) (DeadLetter, error) {
 	raw, err := js.GetMsg(broker.DeadLettersStream, id)
 	if errors.Is(err, nats.ErrMsgNotFound) {
 		return DeadLetter{}, fmt.Errorf("%w: %s holds no message %d — delivered again or dropped already, "+
@@ -286,6 +365,16 @@ func AgainTo(js nats.JetStreamContext, d DeadLetter) (string, error) {
 		// 0363 §7): its consumer on that stream must still be there and still filter the subject, or delivering
 		// it again would hand a module what its grant no longer names. Then it is delivered as an event is, on
 		// EVENTS under the consumer's own again-subject, which the module's consumer there filters.
+		//
+		// **A restricted one only while at least a minute of its stream's age is left** (novox/hq ADR 0376 §5):
+		// past that it can only be dropped, and the copy keeps what is left (DeliverAgain).
+		if d.Restricted {
+			if left := d.AgesOut.Sub(deadLetterNow()); d.AgesOut.IsZero() || left < leastLeftToDeliver {
+				return "", fmt.Errorf("dead letter %d has aged out: %s keeps a message only for its maximum age, and "+
+					"less than a minute of it is left (it ages out at %s), so it is not delivered again. Drop it "+
+					"(novox/hq ADR 0376)", d.ID, d.Stream, d.AgesOut.UTC().Format(time.RFC3339))
+			}
+		}
 		info, err := js.ConsumerInfo(d.Stream, d.Consumer)
 		if errors.Is(err, nats.ErrConsumerNotFound) {
 			return "", fmt.Errorf("dead letter %d was given up by %s, which no longer hears %s, so it would reach a "+
@@ -332,7 +421,16 @@ func AgainTo(js nats.JetStreamContext, d DeadLetter) (string, error) {
 // from DEAD_LETTERS; an ask's original is removed from its work queue before. The message carries its own headers and AgainHeader; its de-duplication id is the
 // kept copy's, so asking twice delivers it once.
 func DeliverAgain(js nats.JetStreamContext, id uint64) (DeadLetter, string, error) {
-	d, err := DeadLetterNamed(js, id)
+	d, to, err := deliverAgain(js, id)
+	if d.Restricted {
+		// What it said stays unsaid to whoever asked it delivered (novox/hq ADR 0376 §4).
+		d.Body, d.Headers, d.Withheld = "", nil, withheldWords
+	}
+	return d, to, err
+}
+
+func deliverAgain(js nats.JetStreamContext, id uint64) (DeadLetter, string, error) {
+	d, err := deadLetterWhole(js, id)
 	if err != nil {
 		return d, "", err
 	}
@@ -349,6 +447,10 @@ func DeliverAgain(js nats.JetStreamContext, id uint64) (DeadLetter, string, erro
 	}
 	again.Header.Set(AgainHeader, strconv.FormatUint(id, 10))
 	again.Header.Set(nats.MsgIdHdr, "again."+broker.DeadLettersStream+"."+strconv.FormatUint(id, 10))
+	if d.Restricted {
+		// The copy keeps what is left of its stream's age (novox/hq ADR 0376 §5).
+		again.Header.Set(nats.MsgTTLHdr, ttlOf(d.AgesOut.Sub(deadLetterNow())))
+	}
 	if _, err := js.PublishMsg(again); err != nil {
 		if d.Original != "" {
 			return d, to, fmt.Errorf("dead letter %d could not be delivered again on %s: %w; it is still kept, so "+
@@ -473,3 +575,56 @@ func keepGivenUp(js nats.JetStreamContext, logger *log.Logger) (*nats.Subscripti
 		_ = m.Ack()
 	}, nats.Bind(broker.DeadLetterNoticesStream, broker.ControllerName), nats.ManualAck())
 }
+
+// Aging is one restricted dead letter as it ages (novox/hq ADR 0376 §6): its id, and when its stream's maximum age
+// runs out and DEAD_LETTERS lets it go.
+type Aging struct {
+	ID      uint64
+	AgesOut time.Time
+}
+
+// AgingDeadLetters is, for each consumer of held whose dead letters came from a restricted stream, each one that
+// ages out and when: read by their headers alone, never their bodies. A consumer of any other stream is left out,
+// and so is a record that keeps no age.
+func AgingDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string][]Aging, error) {
+	out := map[string][]Aging{}
+	restricted := map[string]bool{}
+	for key, n := range held {
+		stream, consumer, _ := strings.Cut(key, ".")
+		is, known := restricted[stream]
+		if !known {
+			r, _, err := sourceAge(js, stream)
+			if err != nil {
+				return nil, err
+			}
+			restricted[stream], is = r, r
+		}
+		if !is || n <= 0 {
+			continue
+		}
+		sub, err := js.SubscribeSync(broker.DeadLetterSubject(stream, consumer), nats.BindStream(broker.DeadLettersStream),
+			nats.OrderedConsumer(), nats.HeadersOnly(), nats.DeliverAll())
+		if err != nil {
+			return nil, fmt.Errorf("the dead letters of %s cannot be read: %w", key, err)
+		}
+		for i := 0; i < n; i++ {
+			m, err := sub.NextMsg(2 * time.Second)
+			if errors.Is(err, nats.ErrTimeout) {
+				break // delivered again, dropped or aged out since they were counted
+			}
+			if err != nil {
+				_ = sub.Unsubscribe()
+				return nil, fmt.Errorf("the dead letters of %s cannot be read: %w", key, err)
+			}
+			meta, err := m.Metadata()
+			if err != nil {
+				continue
+			}
+			if at, err := time.Parse(time.RFC3339Nano, m.Header.Get(DeadHeaderAgesOut)); err == nil {
+				out[key] = append(out[key], Aging{ID: meta.Sequence.Stream, AgesOut: at})
+			}
+		}
+		_ = sub.Unsubscribe()
+	}
+	return out, nil
+}

internal/link/deadletters_restricted_test.go

@@ -0,0 +1,220 @@
+package link
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// A restricted stream of a module, MAIL, kept a week, on the bus (novox/hq ADR 0363), and the subject an event of
+// the account `mesh` is published on.
+func aRestrictedStream(t *testing.T, js *broker.JetStream) string {
+	t.Helper()
+	mail := broker.ModuleStream{Module: "mesh-mail", Name: "MAIL", Events: []string{"mail.*.message.added"},
+		MaxAgeDays: 7, MaxMiB: 16, Restricted: true}
+	_ = js.Context().DeleteStream("MAIL")
+	if err := js.EnsureStream(mail.Stream()); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = js.Context().DeleteStream("MAIL") })
+	return broker.ModuleStreamSubject("mesh-mail", "mail.mesh.message.added")
+}
+
+// withClock sets the clock dead letters are kept and delivered again by, until the test ends.
+func withClock(t *testing.T, now func() time.Time) {
+	t.Helper()
+	before := deadLetterNow
+	deadLetterNow = now
+	t.Cleanup(func() { deadLetterNow = before })
+}
+
+// publishMail puts one mail event on MAIL with a header of its own, and answers its sequence and when it was stored.
+func publishMail(t *testing.T, js *broker.JetStream, subject string) (uint64, time.Time) {
+	t.Helper()
+	msg := &nats.Msg{Subject: subject, Data: []byte(`{"from":"someone@example.org"}`), Header: nats.Header{}}
+	msg.Header.Set("x-account", "mesh")
+	ack, err := js.Context().PublishMsg(msg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := js.Context().GetMsg("MAIL", ack.Sequence)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ack.Sequence, raw.Time
+}
+
+// notice is the server's maximum-deliveries notice for one message of MAIL.
+func notice(seq uint64) []byte {
+	return []byte(fmt.Sprintf(`{"stream":"MAIL","consumer":"ace_mailer","stream_seq":%d,"deliveries":5}`, seq))
+}
+
+// **A dead letter from a restricted stream is listed without its body, and ages out with its stream** (ADR 0376
+// §3, §4): given up on by its consumer and kept, the list and the show by id give what it was and when it ages out,
+// never its body or its own headers, and the kept copy carries a time to live of what is left of MAIL's week.
+func TestARestrictedDeadLetterIsListedWithoutItsBody(t *testing.T) {
+	js := aBus(t)
+	keeping(t, js)
+	subject := aRestrictedStream(t, js)
+	heard := broker.Heard{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true, Filters: []string{subject}}
+	for _, c := range broker.ConsumersOf([]broker.Principal{{Kind: broker.KindModule, Node: "ace", Module: "mailer",
+		Hears: []broker.Heard{heard}, PasswordHash: "x"}}) {
+		if err := js.EnsureConsumer(c.Consumer); err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, published := publishMail(t, js, subject)
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	mailer, err := api.Consumer(t.Context(), "MAIL", "ace_mailer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := givenUp(t, js, mailer)
+	weekLeft := published.Add(7 * 24 * time.Hour)
+	if !d.Restricted || d.Body != "" || d.Headers != nil || d.Subject != subject || d.Sequence != 1 || d.Deliveries != 5 ||
+		d.Published.IsZero() || d.GaveUp.IsZero() || d.Size == 0 || d.AgesOut.Sub(weekLeft).Abs() > time.Second {
+		t.Fatalf("listed as %+v", d)
+	}
+	shown, err := DeadLetterNamed(js.Context(), d.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if shown.Body != "" || shown.Headers != nil || !shown.Restricted || shown.Withheld == "" || shown.AgesOut.IsZero() {
+		t.Fatalf("shown by its id as %+v", shown)
+	}
+	raw, err := js.Context().GetMsg(broker.DeadLettersStream, d.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ttl, err := time.ParseDuration(raw.Header.Get(nats.MsgTTLHdr))
+	if err != nil || ttl > 7*24*time.Hour || ttl < 7*24*time.Hour-time.Minute {
+		t.Fatalf("kept with a time to live of %q (%v), want what is left of the week", raw.Header.Get(nats.MsgTTLHdr), err)
+	}
+	if string(raw.Data) != `{"from":"someone@example.org"}` {
+		t.Fatal("the kept copy does not hold the message, so it could not be delivered again")
+	}
+}
+
+// **Past its stream's age, a restricted dead letter is gone** (ADR 0376 §3), on a fake clock: kept with two seconds
+// of MAIL's week left, DEAD_LETTERS no longer holds it once they pass; with none left, it is kept as a record
+// without its body, which cannot be delivered again.
+func TestARestrictedDeadLetterAgesOutWithItsStream(t *testing.T) {
+	js := aBus(t)
+	subject := aRestrictedStream(t, js)
+	seq, published := publishMail(t, js, subject)
+	withClock(t, func() time.Time { return published.Add(7*24*time.Hour - 2*time.Second) })
+	d, err := KeepDeadLetter(js.Context(), notice(seq))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !d.Restricted || d.Lost != "" {
+		t.Fatalf("kept as %+v", d)
+	}
+	eventually(t, "the dead letter aged out", func() bool {
+		_, err := js.Context().GetMsg(broker.DeadLettersStream, d.ID)
+		return err != nil
+	})
+	if held, _ := HeldDeadLetters(js.Context()); held["MAIL.ace_mailer"] != 0 {
+		t.Fatalf("still held after its age: %v", held)
+	}
+
+	seq, published = publishMail(t, js, subject)
+	withClock(t, func() time.Time { return published.Add(7*24*time.Hour + time.Minute) })
+	past, err := KeepDeadLetter(js.Context(), notice(seq))
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := js.Context().GetMsg(broker.DeadLettersStream, past.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(raw.Data) != 0 || raw.Header.Get("x-account") != "" || !strings.Contains(past.Lost, "maximum age") {
+		t.Fatalf("kept past its age as %+v with %q", past, raw.Data)
+	}
+	if _, _, err := DeliverAgain(js.Context(), past.ID); err == nil || !strings.Contains(err.Error(), "Drop it") {
+		t.Fatalf("a record past its age was delivered again: %v", err)
+	}
+}
+
+// **A restricted dead letter is delivered again only within its age, and the copy keeps that age** (ADR 0376 §5):
+// with more than a minute left the copy goes on EVENTS under the consumer's again subject with a time to live of
+// what is left; with less, it is refused as aged out.
+func TestARestrictedDeadLetterGoesBackOnlyWithinItsAge(t *testing.T) {
+	js := aBus(t)
+	subject := aRestrictedStream(t, js)
+	heard := broker.Heard{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true, Filters: []string{subject}}
+	for _, c := range broker.ConsumersOf([]broker.Principal{{Kind: broker.KindModule, Node: "ace", Module: "mailer",
+		Hears: []broker.Heard{heard}, PasswordHash: "x"}}) {
+		if err := js.EnsureConsumer(c.Consumer); err != nil {
+			t.Fatal(err)
+		}
+	}
+	seq, published := publishMail(t, js, subject)
+	at := published.Add(7*24*time.Hour - 10*time.Minute)
+	withClock(t, func() time.Time { return at })
+	d, err := KeepDeadLetter(js.Context(), notice(seq))
+	if err != nil {
+		t.Fatal(err)
+	}
+	at = published.Add(7*24*time.Hour - 30*time.Second)
+	if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "aged out") {
+		t.Fatalf("delivered again with under a minute left: %v", err)
+	}
+	at = published.Add(7*24*time.Hour - 5*time.Minute)
+	_, to, err := DeliverAgain(js.Context(), d.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if to != broker.AgainSubject("ace_mailer", subject) {
+		t.Fatalf("delivered again on %s", to)
+	}
+	raw, err := js.Context().GetLastMsg(broker.EventsStream, to)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ttl, err := time.ParseDuration(raw.Header.Get(nats.MsgTTLHdr))
+	if err != nil || ttl > 5*time.Minute || ttl < 4*time.Minute {
+		t.Fatalf("the copy carries a time to live of %q (%v), want what is left", raw.Header.Get(nats.MsgTTLHdr), err)
+	}
+	if string(raw.Data) != `{"from":"someone@example.org"}` || raw.Header.Get("x-account") != "mesh" {
+		t.Fatalf("the copy is %q %v", raw.Data, raw.Header)
+	}
+}
+
+// What of a consumer's dead letters ages out, and when, is read without their bodies, and only for a restricted
+// stream's (ADR 0376 §6).
+func TestTheAgingOfRestrictedDeadLettersIsRead(t *testing.T) {
+	js := aBus(t)
+	subject := aRestrictedStream(t, js)
+	seq, published := publishMail(t, js, subject)
+	withClock(t, func() time.Time { return published.Add(time.Hour) })
+	d, err := KeepDeadLetter(js.Context(), notice(seq))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
+		`"deliveries":5}`)); err != nil {
+		t.Fatal(err)
+	}
+	held, err := HeldDeadLetters(js.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	aging, err := AgingDeadLetters(js.Context(), held)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(aging) != 1 || len(aging["MAIL.ace_mailer"]) != 1 || aging["MAIL.ace_mailer"][0].ID != d.ID ||
+		!aging["MAIL.ace_mailer"][0].AgesOut.Equal(d.AgesOut) {
+		t.Fatalf("aging %+v, kept %+v", aging, d)
+	}
+}