ADR 0590 supersedes the lease write: mesh-grants says no lease event any more, so the watchman's lease-altered and removal-failed findings and their row had nothing to hear. The ended controller's consumers are removed as hand acts, so their deletion is no consumer lost; only the one on DEAD_LETTER_NOTICES is still read, since its removal hands the notices to the watchman. The builder hears artifacts-gone from the artifact sweep seat alone, and mesh-enrolment's tokens no longer report an issuing gate that nothing can turn off. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
352 lines
15 KiB
Go
352 lines
15 KiB
Go
// Package deadletters is what a consumer gave up on, kept by the watchman since the controller's end (novox/hq ADR 0564 §5,
|
|
// ADR 0264, ADR 0376, issue 1952 WM5): ported from mesh-controller internal/link/deadletters.go at e04846e9 with nothing
|
|
// changed in what is kept, aged or withheld, on the jetstream API.
|
|
//
|
|
// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The ended controller's consumer `controller` never
|
|
// expires, so the watchman makes its own (Consumer) only once `controller` is gone (Gate); while it stands the watchman says
|
|
// so as a finding. Removing `controller` is a hand act with the bus module's tool (novox/hq ADR 0590 clause 6). Delivering
|
|
// and dropping wait for the hand-act log's home (issue 1953), so they are refused naming that.
|
|
package deadletters
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
|
)
|
|
|
|
// Consumer is the watchman's own consumer on DEAD_LETTER_NOTICES, made only by Gate's leave.
|
|
const Consumer = "mesh-delivery-watchman"
|
|
|
|
// The headers a kept message carries beside its own (mesh-controller link.DeadHeader*: the same names, so the
|
|
// controller's copies and the watchman's are read alike).
|
|
const (
|
|
HeaderStream = "Mesh-Dead-Stream"
|
|
HeaderConsumer = "Mesh-Dead-Consumer"
|
|
HeaderSequence = "Mesh-Dead-Sequence"
|
|
HeaderSubject = "Mesh-Dead-Subject"
|
|
HeaderDeliveries = "Mesh-Dead-Deliveries"
|
|
HeaderGaveUp = "Mesh-Dead-Gave-Up"
|
|
HeaderPublished = "Mesh-Dead-Published"
|
|
HeaderLost = "Mesh-Dead-Lost"
|
|
HeaderMsgID = "Mesh-Dead-Msg-Id"
|
|
HeaderRestricted = "Mesh-Dead-Restricted"
|
|
HeaderAgesOut = "Mesh-Dead-Ages-Out"
|
|
HeaderWorkQueue = "Mesh-Dead-Work-Queue"
|
|
)
|
|
|
|
// DeadLetter is one message a consumer gave up on, as DEAD_LETTERS keeps it (the controller's JSON).
|
|
type DeadLetter struct {
|
|
ID uint64 `json:"id"`
|
|
Stream string `json:"stream"`
|
|
Consumer string `json:"consumer"`
|
|
Who string `json:"who"`
|
|
Subject string `json:"subject"`
|
|
Sequence uint64 `json:"sequence"`
|
|
Deliveries uint64 `json:"deliveries"`
|
|
GaveUp time.Time `json:"gave_up"`
|
|
Published time.Time `json:"published,omitzero"`
|
|
Lost string `json:"lost,omitempty"`
|
|
Size int `json:"size"`
|
|
Restricted bool `json:"restricted,omitempty"`
|
|
AgesOut time.Time `json:"ages_out,omitzero"`
|
|
Withheld string `json:"withheld,omitempty"`
|
|
WorkQueue bool `json:"work-queue,omitempty"`
|
|
Body string `json:"body,omitempty"`
|
|
Headers map[string][]string `json:"headers,omitempty"`
|
|
}
|
|
|
|
// Now is the clock dead letters age by: a variable, so a test can move it.
|
|
var Now = time.Now
|
|
|
|
// WithheldWords is what is said in place of a restricted dead letter's body.
|
|
const WithheldWords = "its body and its own headers are not shown: it came from a restricted stream, which only " +
|
|
"the modules its grant names may read (novox/hq ADR 0376)"
|
|
|
|
// ConsumerInWords is a durable consumer as the mesh says it: whose, and for what.
|
|
func ConsumerInWords(stream, name string) string {
|
|
switch {
|
|
case name == buscompose.ControllerName || name == buscompose.MergesConsumerName:
|
|
return "the controller's consumer on " + stream
|
|
case stream == "NODES":
|
|
return "how " + name + " hears what it should be (its declaration consumer)"
|
|
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
|
|
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
|
|
return "the worker every holder of " + seat + " takes its asks from"
|
|
case stream == buscompose.EventsStream:
|
|
if node, module, ok := strings.Cut(name, "_"); ok {
|
|
return "how " + module + " on " + node + " hears what it consumes"
|
|
}
|
|
}
|
|
return "the consumer " + name + " on " + stream
|
|
}
|
|
|
|
// sourceAge is whether a stream is restricted, a module's declared work queue, and its maximum age; a stream gone is
|
|
// neither.
|
|
func sourceAge(ctx context.Context, js jetstream.JetStream, stream string) (restricted, workQueue bool, maxAge time.Duration, err error) {
|
|
s, err := js.Stream(ctx, stream)
|
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
|
return false, false, 0, nil
|
|
}
|
|
if err != nil {
|
|
return false, false, 0, fmt.Errorf("whether %s is restricted cannot be read: %w", stream, err)
|
|
}
|
|
cfg := s.CachedInfo().Config
|
|
return cfg.Metadata[buscompose.RestrictedMetadata] == "true", cfg.Metadata[buscompose.WorkQueueMetadata] == "true", cfg.MaxAge, nil
|
|
}
|
|
|
|
// notice is the part of the server's maximum-deliveries advisory a kept message is made from.
|
|
type notice struct {
|
|
Stream string `json:"stream"`
|
|
Consumer string `json:"consumer"`
|
|
StreamSeq uint64 `json:"stream_seq"`
|
|
Deliveries uint64 `json:"deliveries"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
// Keep keeps the message one maximum-deliveries notice is about. An error keeps nothing and the notice is to be taken
|
|
// again; a source that no longer holds the message is no error: the record is kept with why.
|
|
func Keep(ctx context.Context, js jetstream.JetStream, body []byte) (DeadLetter, error) {
|
|
var a notice
|
|
if err := json.Unmarshal(body, &a); err != nil || a.Stream == "" || a.Consumer == "" || a.StreamSeq == 0 {
|
|
return DeadLetter{}, fmt.Errorf("not a maximum-deliveries notice the mesh can read: %.200s", body)
|
|
}
|
|
gaveUp := a.Timestamp
|
|
if gaveUp.IsZero() {
|
|
gaveUp = Now()
|
|
}
|
|
kept := &nats.Msg{Subject: buscompose.DeadLetterSubject(a.Stream, a.Consumer), Header: nats.Header{}}
|
|
restricted, workQueue, maxAge, err := sourceAge(ctx, js, a.Stream)
|
|
if err != nil {
|
|
return DeadLetter{}, err
|
|
}
|
|
if restricted {
|
|
kept.Header.Set(HeaderRestricted, "true")
|
|
}
|
|
if workQueue {
|
|
kept.Header.Set(HeaderWorkQueue, "true")
|
|
}
|
|
var raw *jetstream.RawStreamMsg
|
|
src, err := js.Stream(ctx, a.Stream)
|
|
if err == nil {
|
|
raw, err = src.GetMsg(ctx, a.StreamSeq)
|
|
}
|
|
agesOut := time.Time{}
|
|
if err == nil && restricted && maxAge > 0 {
|
|
agesOut = raw.Time.Add(maxAge)
|
|
kept.Header.Set(HeaderAgesOut, agesOut.UTC().Format(time.RFC3339Nano))
|
|
}
|
|
switch {
|
|
case err == nil && !agesOut.IsZero() && agesOut.Sub(Now()) < time.Second:
|
|
kept.Header.Set(HeaderSubject, raw.Subject)
|
|
kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano))
|
|
kept.Header.Set(HeaderLost, fmt.Sprintf("message %d of %s had passed its stream's maximum age of %s when the notice "+
|
|
"was taken, so it is kept without its body (novox/hq ADR 0376)", a.StreamSeq, a.Stream, maxAge))
|
|
case err == nil:
|
|
for k, v := range raw.Header {
|
|
if !strings.HasPrefix(k, "Nats-") {
|
|
kept.Header[k] = v
|
|
}
|
|
}
|
|
if id := raw.Header.Get(nats.MsgIdHdr); id != "" {
|
|
kept.Header.Set(HeaderMsgID, id)
|
|
}
|
|
kept.Header.Set(HeaderSubject, raw.Subject)
|
|
kept.Header.Set(HeaderPublished, raw.Time.UTC().Format(time.RFC3339Nano))
|
|
kept.Data = raw.Data
|
|
if !agesOut.IsZero() {
|
|
kept.Header.Set(nats.MsgTTLHdr, strconv.FormatInt(int64(agesOut.Sub(Now())/time.Second), 10)+"s")
|
|
}
|
|
case errors.Is(err, jetstream.ErrMsgNotFound) || errors.Is(err, jetstream.ErrStreamNotFound):
|
|
kept.Header.Set(HeaderLost, fmt.Sprintf("%s no longer held message %d when the notice was taken: %v", a.Stream, a.StreamSeq, err))
|
|
default:
|
|
return DeadLetter{}, fmt.Errorf("message %d of %s could not be read: %w", a.StreamSeq, a.Stream, err)
|
|
}
|
|
kept.Header.Set(HeaderStream, a.Stream)
|
|
kept.Header.Set(HeaderConsumer, a.Consumer)
|
|
kept.Header.Set(HeaderSequence, strconv.FormatUint(a.StreamSeq, 10))
|
|
kept.Header.Set(HeaderDeliveries, strconv.FormatUint(a.Deliveries, 10))
|
|
kept.Header.Set(HeaderGaveUp, gaveUp.UTC().Format(time.RFC3339Nano))
|
|
// One copy per message given up, however often its notice is taken, by whichever keeper takes it.
|
|
kept.Header.Set(nats.MsgIdHdr, fmt.Sprintf("%s.%s.%d", a.Stream, a.Consumer, a.StreamSeq))
|
|
ack, err := js.PublishMsg(ctx, kept)
|
|
if err != nil {
|
|
return DeadLetter{}, fmt.Errorf("message %d of %s could not be kept in %s: %w", a.StreamSeq, a.Stream, buscompose.DeadLettersStream, err)
|
|
}
|
|
return Of(ack.Sequence, kept.Subject, kept.Header, kept.Data, false), nil
|
|
}
|
|
|
|
// Of reads a kept message; whole gives its body and its own headers.
|
|
func Of(id uint64, subject string, h nats.Header, data []byte, whole bool) DeadLetter {
|
|
d := DeadLetter{ID: id, Stream: h.Get(HeaderStream), Consumer: h.Get(HeaderConsumer), Subject: h.Get(HeaderSubject),
|
|
Lost: h.Get(HeaderLost), Size: len(data)}
|
|
if d.Stream == "" || d.Consumer == "" {
|
|
d.Stream, d.Consumer, _ = buscompose.DeadLetterOf(subject)
|
|
}
|
|
d.Who = ConsumerInWords(d.Stream, d.Consumer)
|
|
d.Sequence, _ = strconv.ParseUint(h.Get(HeaderSequence), 10, 64)
|
|
d.Deliveries, _ = strconv.ParseUint(h.Get(HeaderDeliveries), 10, 64)
|
|
d.GaveUp, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderGaveUp))
|
|
d.Published, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderPublished))
|
|
d.Restricted = h.Get(HeaderRestricted) == "true"
|
|
d.WorkQueue = h.Get(HeaderWorkQueue) == "true"
|
|
d.AgesOut, _ = time.Parse(time.RFC3339Nano, h.Get(HeaderAgesOut))
|
|
if whole {
|
|
d.Body = string(data)
|
|
d.Headers = map[string][]string{}
|
|
for k, v := range h {
|
|
if !strings.HasPrefix(k, "Mesh-Dead-") && !strings.HasPrefix(k, "Nats-") {
|
|
d.Headers[k] = append([]string(nil), v...)
|
|
}
|
|
}
|
|
}
|
|
return d
|
|
}
|
|
|
|
// Held is how many messages DEAD_LETTERS holds for each consumer, by `<stream>.<consumer>`.
|
|
func Held(ctx context.Context, js jetstream.JetStream) (map[string]int, error) {
|
|
s, err := js.Stream(ctx, buscompose.DeadLettersStream)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err)
|
|
}
|
|
info, err := s.Info(ctx, jetstream.WithSubjectFilter(">"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err)
|
|
}
|
|
held := map[string]int{}
|
|
for subject, n := range info.State.Subjects {
|
|
if stream, consumer, ok := buscompose.DeadLetterOf(subject); ok && n > 0 {
|
|
held[stream+"."+consumer] += int(n)
|
|
}
|
|
}
|
|
return held, nil
|
|
}
|
|
|
|
// List is what DEAD_LETTERS holds, newest first, at most most (all when most is not positive); for one consumer when
|
|
// consumer names one (its name, or `<stream>.<consumer>`). A restricted one's body is withheld. The total is the stream's
|
|
// own count.
|
|
func List(ctx context.Context, js jetstream.JetStream, consumer string, most int) ([]DeadLetter, int, error) {
|
|
held, err := Held(ctx, js)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
total := 0
|
|
for key, n := range held {
|
|
if consumer == "" || key == consumer || strings.HasSuffix(key, "."+consumer) {
|
|
total += n
|
|
}
|
|
}
|
|
if total == 0 {
|
|
return nil, 0, nil
|
|
}
|
|
s, err := js.Stream(ctx, buscompose.DeadLettersStream)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
info, err := s.Info(ctx)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("%s cannot be read: %w", buscompose.DeadLettersStream, err)
|
|
}
|
|
var out []DeadLetter
|
|
for seq := info.State.LastSeq; seq >= info.State.FirstSeq && seq > 0; seq-- {
|
|
if most > 0 && len(out) >= most || len(out) >= total {
|
|
break
|
|
}
|
|
raw, err := s.GetMsg(ctx, seq)
|
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return out, total, fmt.Errorf("dead letter %d cannot be read: %w", seq, err)
|
|
}
|
|
d := Of(seq, raw.Subject, raw.Header, raw.Data, false)
|
|
if consumer != "" && consumer != d.Consumer && consumer != d.Stream+"."+d.Consumer {
|
|
continue
|
|
}
|
|
if d.Restricted {
|
|
d.Withheld = WithheldWords
|
|
}
|
|
out = append(out, d)
|
|
}
|
|
return out, total, nil
|
|
}
|
|
|
|
// ErrNoDeadLetter is an id DEAD_LETTERS does not hold.
|
|
var ErrNoDeadLetter = errors.New("no such dead letter")
|
|
|
|
// Named is one kept message whole, but for a restricted one, whose body and own headers are never shown.
|
|
func Named(ctx context.Context, js jetstream.JetStream, id uint64) (DeadLetter, error) {
|
|
s, err := js.Stream(ctx, buscompose.DeadLettersStream)
|
|
if err != nil {
|
|
return DeadLetter{}, err
|
|
}
|
|
raw, err := s.GetMsg(ctx, id)
|
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
|
return DeadLetter{}, fmt.Errorf("%w: %s holds no message %d — delivered again or dropped already, or never kept; "+
|
|
"dead-letters lists what it holds", ErrNoDeadLetter, buscompose.DeadLettersStream, id)
|
|
}
|
|
if err != nil {
|
|
return DeadLetter{}, fmt.Errorf("dead letter %d cannot be read: %w", id, err)
|
|
}
|
|
d := Of(id, raw.Subject, raw.Header, raw.Data, true)
|
|
if d.Restricted {
|
|
d.Body, d.Headers, d.Withheld = "", nil, WithheldWords
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// Refusal is why the watchman delivers nothing again and drops nothing: the act waits for the hand-act log's home (issue
|
|
// 1953), where a person's act is recorded. act is deliver or drop.
|
|
func Refusal(act string) error {
|
|
done := map[string]string{"deliver": "delivered again", "drop": "dropped"}[act]
|
|
if done == "" {
|
|
return fmt.Errorf("%q is not deliver or drop", act)
|
|
}
|
|
return fmt.Errorf("a dead letter is %s only as a person's act recorded in the hand-act log, whose home is not "+
|
|
"delivered yet (issue 1953): nothing is %s", done, done)
|
|
}
|
|
|
|
// Gate is whether the watchman takes the notices itself: once the controller's consumer is gone. waiting says `controller`
|
|
// still stands, which is the watchman's finding: each notice waits in the queue until that consumer is removed (the
|
|
// controller ended by its unassignment, issue 1126).
|
|
func Gate(controllerStands bool) (take, waiting bool) {
|
|
return !controllerStands, controllerStands
|
|
}
|
|
|
|
// ControllerStands asks the bus whether the controller's consumer on DEAD_LETTER_NOTICES stands.
|
|
func ControllerStands(ctx context.Context, js jetstream.JetStream) (bool, error) {
|
|
_, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, buscompose.ControllerName)
|
|
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
|
return false, nil
|
|
}
|
|
return err == nil, err
|
|
}
|
|
|
|
// TakeNotices makes the watchman's own durable consumer on DEAD_LETTER_NOTICES (never touching `controller`) and keeps
|
|
// each notice it takes; a notice not kept is handed back to be taken again. kept hears each one kept.
|
|
func TakeNotices(ctx context.Context, js jetstream.JetStream, kept func(DeadLetter), logf func(string, ...any)) (jetstream.ConsumeContext, error) {
|
|
c, err := js.CreateOrUpdateConsumer(ctx, buscompose.DeadLetterNoticesStream, jetstream.ConsumerConfig{
|
|
Durable: Consumer, AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Minute})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the consumer %s on %s cannot be made: %w", Consumer, buscompose.DeadLetterNoticesStream, err)
|
|
}
|
|
return c.Consume(func(m jetstream.Msg) {
|
|
d, err := Keep(ctx, js, m.Data())
|
|
if err != nil {
|
|
logf("a dead-letter notice was not kept, offered again in a minute: %v", err)
|
|
_ = m.NakWithDelay(time.Minute)
|
|
return
|
|
}
|
|
_ = m.Ack()
|
|
kept(d)
|
|
})
|
|
}
|