novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Let a module stream declare work-queue retention (issue 1177)

Browse Source
This commit is contained in:
jochen
2026-10-19 02:11:05 +02:00
parent ffad53aad2
commit a069898c0c
6 changed files with 206 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/catalogue/stream_work_queue_test.go
+66
View File
@@ -0,0 +1,66 @@
package catalogue
import (
"strings"
"testing"
)
// A module stream may declare retention: work-queue (novox/hq ADR 0436 §6): a message is removed once a
// consumer acknowledges it, the bus refusing a new one rather than dropping what it holds. The fixture is
// mesh-mail's future `mail-sends` stream (ADR 0436 §7, §8), copied from the module.json shape at
// /home/jochen/projects/novox/mesh-catalog/modules/mesh-mail/module.json and extended with retention,
// max-deliveries and duplicates as the ADR's clause 6 describes.
const mailSendsStream = `[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":64,` +
`"retention":"work-queue","max-deliveries":5,"duplicates":24}]`
// 1177/AC1: a well-formed work-queue stream is accepted and read back whole.
func TestAWorkQueueStreamIsDeclaredAndAccepted(t *testing.T) {
m, err := ParseManifest([]byte(mailManifest(t, mailSendsStream, "")))
if err != nil {
t.Fatalf("a well-formed work-queue stream was refused: %v", err)
}
s := m.Streams[0]
if s.Retention != StreamRetentionWorkQueue || s.MaxDeliveries != 5 || s.Duplicates != 24 ||
s.Days() != 1 || s.Cap() != 64 {
t.Fatalf("the stream read as %+v", s)
}
}
// 1177/AC2: a stream whose work-queue values are out of range, contradictory, or carries a grant is refused
// naming the stream and the rule.
func TestAWorkQueueStreamsValuesAreBounded(t *testing.T) {
for name, c := range map[string]struct{ streams, says string }{
"max-deliveries 0": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
`"max-deliveries":0,"duplicates":24}]`, "between 1 and 10"},
"max-deliveries 11": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"work-queue",` +
`"max-deliveries":11,"duplicates":24}]`, "between 1 and 10"},
"duplicates longer than its maximum age": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"max-age-days":1,"retention":"work-queue","max-deliveries":5,"duplicates":25}]`, "longer than its maximum age"},
"an unknown retention": {`[{"name":"mail-sends","events":["mail.send.approved"],"retention":"sometimes"}]`,
"retention"},
"max-deliveries without retention work-queue": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"max-deliveries":5}]`, "without retention"},
"duplicates without retention work-queue": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"duplicates":24}]`, "without retention"},
"a work-queue stream carrying a grant": {`[{"name":"mail-sends","events":["mail.send.approved"],` +
`"retention":"work-queue","max-deliveries":5,` +
`"grant":{"config":"grants","events":"mail-sends.{key}.**"}}]`, "carrying a grant"},
} {
_, err := ParseManifest([]byte(mailManifest(t, c.streams, "")))
if err == nil || !strings.Contains(err.Error(), "mail-sends") || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: %v; want a refusal naming the stream and %q", name, err, c.says)
}
}
}
// 1177/AC3: a stream with no retention composes as main does — limits retention, dropping its oldest — proven
// at the broker, where the bus's retention policy is decided (internal/broker/modstreams_test.go).
func TestAStreamWithNoRetentionReadsAsLimits(t *testing.T) {
m, err := ParseManifest([]byte(mailManifest(t, mailStream, "")))
if err != nil {
t.Fatal(err)
}
if got := m.Streams[0].Retention; got != "" {
t.Fatalf("a stream declaring no retention read %q", got)
}
}
internal/catalogue/streams.go
+61 -1
View File
@@ -30,6 +30,18 @@ const (
// GrantKey is the name in a grant's pattern that each key of the config value fills.
const GrantKey = "{key}"
// StreamRetentionWorkQueue is a stream's declared retention (novox/hq ADR 0436 §6): a message is removed
// once a consumer acknowledges it, and the bus refuses a new one rather than dropping what it holds. Empty
// is the mesh's own limits retention — kept until it ages or the stream fills, dropping its oldest.
const StreamRetentionWorkQueue = "work-queue"
// StreamLeastDeliveries and StreamMostDeliveries bound a work-queue stream's max-deliveries (novox/hq ADR
// 0436 §6): how many times a message is handed over before it is given up as a dead letter.
const (
StreamLeastDeliveries = 1
StreamMostDeliveries = 10
)
// ReservedStreamPrefixes are the starts of the stream names the mesh derives, which no module's stream
// may take: a seat's work queue, a log, a bucket, an object store, and the dead letters.
var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
@@ -48,6 +60,15 @@ type StreamDeclaration struct {
MaxMiB int `json:"max-mib,omitempty"`
// Grant restricts it, when said.
Grant *StreamGrant `json:"grant,omitempty"`
// Retention is how the stream decides what to keep when it fills (novox/hq ADR 0436 §6): empty for the
// mesh's own limits retention (dropping its oldest), or StreamRetentionWorkQueue.
Retention string `json:"retention,omitempty"`
// MaxDeliveries bounds how many times a work-queue message is handed over, 1 to 10; refused without
// retention work-queue (novox/hq ADR 0436 §6).
MaxDeliveries int `json:"max-deliveries,omitempty"`
// Duplicates is a work-queue stream's duplicate window in hours, at most its maximum age; refused
// without retention work-queue (novox/hq ADR 0436 §6).
Duplicates int `json:"duplicates,omitempty"`
}
// StreamGrant says which trusted config value of the module grants a consumer a part of the stream, and
@@ -67,7 +88,8 @@ func (s *StreamDeclaration) UnmarshalJSON(raw []byte) error {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&p); err != nil {
return fmt.Errorf("a stream is {name, events, max-age-days, max-mib, grant}: %w", typedUnknown(err))
return fmt.Errorf("a stream is {name, events, max-age-days, max-mib, grant, retention, max-deliveries, "+
"duplicates}: %w", typedUnknown(err))
}
*s = StreamDeclaration(p)
return nil
@@ -163,6 +185,44 @@ func StreamProblems(m Manifest) []string {
problems = append(problems, fmt.Sprintf("%s caps stream %q at %d MiB; a stream holds between %d and %d "+
"(novox/hq ADR 0363)", m.Module, s.Name, s.MaxMiB, StreamLeastMiB, StreamMostMiB))
}
// A stream may declare retention: work-queue (novox/hq ADR 0436 §6): a message removed once a
// consumer acknowledges it, the bus refusing a new one when full rather than dropping its oldest, with
// max-deliveries (1 to 10) and a duplicate window (duplicates, hours, at most its maximum age) -- both
// meaningless, and refused, without it. A work-queue stream is read by its one consumer, never granted.
switch s.Retention {
case "", StreamRetentionWorkQueue:
default:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares retention %q; a stream's retention is %q or nothing (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.Retention, StreamRetentionWorkQueue))
}
switch {
case s.Retention == StreamRetentionWorkQueue &&
(s.MaxDeliveries < StreamLeastDeliveries || s.MaxDeliveries > StreamMostDeliveries):
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares max-deliveries %d; a work-queue stream's max-deliveries is between %d "+
"and %d (novox/hq ADR 0436 §6)", m.Module, s.Name, s.MaxDeliveries, StreamLeastDeliveries,
StreamMostDeliveries))
case s.Retention != StreamRetentionWorkQueue && s.MaxDeliveries != 0:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares max-deliveries %d without retention %q (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.MaxDeliveries, StreamRetentionWorkQueue))
}
switch {
case s.Duplicates != 0 && s.Retention != StreamRetentionWorkQueue:
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares duplicates %d hours without retention %q (novox/hq ADR 0436 §6)",
m.Module, s.Name, s.Duplicates, StreamRetentionWorkQueue))
case s.Duplicates != 0 && s.Duplicates > s.Days()*24:
problems = append(problems, fmt.Sprintf(
"%s's stream %q keeps duplicates %d hours, longer than its maximum age of %d days (novox/hq "+
"ADR 0436 §6)", m.Module, s.Name, s.Duplicates, s.Days()))
}
if s.Retention == StreamRetentionWorkQueue && s.Grant != nil {
problems = append(problems, fmt.Sprintf(
"%s's stream %q declares retention %q carrying a grant; a work-queue stream is read by its "+
"module's one consumer, never granted (novox/hq ADR 0436 §6)", m.Module, s.Name, s.Retention))
}
if len(s.Events) == 0 {
problems = append(problems, fmt.Sprintf("%s declares stream %q with no events", m.Module, s.Name))
}