nats: nats_consumer_remove removes only a durable that is itself an orphan

Review round 1: a durable whose twin is current was removed without
reading the durable itself, so of the five live pairs where both twins
have pulls waiting either one would have gone. The tool now reads the
named durable too and refuses it while pulls wait on it, both read from
the bus at the call. "Current" is the setting twin-current-hours (24 by
default) of the manifest, read from the configuration at each call, the
field name the grants part's manifest names too.

Builds: design 47 H2 H-A-472 (1) and its third addendum (1), (2); review round 1 findings (1), (2)
This commit is contained in:
2026-11-22 14:33:01 +01:00
parent e0bd0bedc4
commit 3443760205
6 changed files with 116 additions and 48 deletions
+1 -1
View File
@@ -191,7 +191,7 @@ func allTools() []stdio.Tool {
defer cancel()
return monitor(ctx, heldJsz)
}
out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat)...)
out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat, bundleTwinCurrent)...)
out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...)
out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...)
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
@@ -346,6 +346,11 @@ func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) {
if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 {
t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config)
}
// Design 47 H2, H-A-472 third addendum (2): twin-current-hours, the bound of a current twin, defaults to 24, the field
// name the grants part's manifest names too.
if m.Config["twin-current-hours"].Default != DefaultTwinCurrentHours || DefaultTwinCurrentHours != 24 {
t.Errorf("config %v: twin-current-hours defaults to 24", m.Config)
}
found := false
for _, r := range m.Resources {
if r.ID == "keeper-settings" {
+50 -16
View File
@@ -13,6 +13,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
@@ -347,9 +348,13 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an
// every module it heard as accepted, its state plans, one key per module of a watched branch's newest revision whether
// it asks for kept data or not, design 47 G §538) is refused, on any stream; every other durable is an orphan and goes, `why` its warrant, at once, and its removal is
// recorded as a hand act through mesh-hand-acts, as nats_bucket_expiry's change is.
func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded) []stdio.Tool {
func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded, twinWithin func() time.Duration) []stdio.Tool {
removeConsumer := func(a map[string]any) (any, error) {
twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now())
within := DefaultTwinCurrentHours * time.Hour
if twinWithin != nil {
within = twinWithin()
}
twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now(), within)
if err != nil {
return nil, err
}
@@ -384,7 +389,7 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu
}
return []stdio.Tool{
{Name: "nats_consumer_remove",
Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named <node>_<module> or beginning <node>_<module>_ for a module of an accepted manifest, on any stream, unless it is an orphan: its twin (<node>_<module> for <node>_<module>_tools, and back) on the same stream is current, with pulls waiting and a delivery within the day, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.",
Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named <node>_<module> or beginning <node>_<module>_ for a module of an accepted manifest, on any stream, unless it is an orphan: no pulls waiting on it while its twin (<node>_<module> for <node>_<module>_tools, and back) on the same stream is current, with pulls waiting and a delivery within the setting twin-current-hours (24 by default), both read at the call, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.",
Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")},
Run: removeConsumer},
{Name: "nats_stream_remove",
@@ -410,12 +415,14 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu
// module keeping none would lose its live durables: H-A-128's silent failure, P3). A set that cannot be read refuses
// too: what cannot be told apart is not removed.
//
// **An orphaned durable goes** (design 47 H2, H-A-472 (1)): a module's durable whose live twin on the same stream exists
// and is current is not refused, and the twin's name is answered, for the hand act to name it. The twins are the
// **An orphaned durable goes** (design 47 H2, H-A-472 (1) and its third addendum): a module's durable that is itself an
// orphan (no pulls waiting on it) while its live twin on the same stream exists and is current is not refused, and the
// twin's name is answered, for the hand act to name it; both are read from the bus at the call. The twins are the
// module's program's durable <node>_<module> and its tool runner's <node>_<module>_tools (design 47 G, A-340): a module
// that moved into the tool runner left the first, one that runs its own program again left the second. A module's
// only durable, any other suffix, and a twin that cannot be read or is not current, stay refused (H-A-127/128).
func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time) (string, error) {
// only durable, any other suffix, a durable with pulls waiting, and a durable or twin that cannot be read or a twin
// that is not current, stay refused (H-A-127/128). within is the bound of "current", the setting twin-current-hours.
func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time, within time.Duration) (string, error) {
if accepted == nil {
return "", nil
}
@@ -438,17 +445,26 @@ func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModule
}
js, closeBus, err := bus(ctx)
if err != nil {
return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err)
return "", fmt.Errorf("%v (it and its twin %s on %s cannot be read: %v)", refused, twin, stream, err)
}
defer closeBus()
sctx, cancel := step(ctx)
self, err := readConsumer(sctx, js, stream, consumer)
cancel()
if err != nil {
return "", fmt.Errorf("%v (it cannot be read on %s: %v)", refused, stream, err)
}
if self.NumWaiting > 0 {
return "", fmt.Errorf("%v (it is no orphan: %d pulls waiting on it)", refused, self.NumWaiting)
}
sctx, cancel = step(ctx)
info, err := readConsumer(sctx, js, stream, twin)
cancel()
closeBus()
if err != nil {
return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err)
}
if !current(info, now) {
return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info))
if !current(info, now, within) {
return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s, the bound %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info), within)
}
return twin, nil
}
@@ -466,12 +482,30 @@ func twinOf(node, module, durable string) string {
return ""
}
// currentWithin is how recent a twin's last delivery is for it to be current: within the day (H-A-472 (1)).
const currentWithin = 24 * time.Hour
// DefaultTwinCurrentHours is the setting twin-current-hours's default (design 47 H2, H-A-472 third addendum): how recent a
// twin's last delivery is for it to be current, within the day; the grants part reads the same field name of its own
// manifest for its orphaned durables.
const DefaultTwinCurrentHours = 24
// current says whether a durable is live: pulls waiting on it and a delivery within the day (H-A-472 (1)).
func current(info *jetstream.ConsumerInfo, now time.Time) bool {
return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < currentWithin
// twinCurrentWithin is the setting twin-current-hours as the configuration answers it, in hours, as a bound: its default
// when the configuration gives none, or a value that is not a positive whole number.
func twinCurrentWithin(raw json.RawMessage, err error) time.Duration {
var n int
if err != nil || json.Unmarshal(raw, &n) != nil || n < 1 {
n = DefaultTwinCurrentHours
}
return time.Duration(n) * time.Hour
}
// bundleTwinCurrent is the setting twin-current-hours as this bundle's configuration holds it, read at each call
// (never while the tools are listed: see bundleKeepDays).
func bundleTwinCurrent() time.Duration {
return twinCurrentWithin(stdio.Config("twin-current-hours"))
}
// current says whether a durable is live: pulls waiting on it and a delivery within the bound (H-A-472 (1)).
func current(info *jetstream.ConsumerInfo, now time.Time, within time.Duration) bool {
return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < within
}
func lastDelivery(info *jetstream.ConsumerInfo) string {
+25 -20
View File
@@ -197,7 +197,7 @@ accounts { MESH { jetstream: enabled, users: [
// Through the tool, `why` its warrant: each removal is recorded as a hand act (design 47 H, H-A-127).
var handActs []meshhandactsclient.Request
var removeConsumer func(map[string]any) (any, error)
for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) {
for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) {
if tl.Name == "nats_consumer_remove" {
removeConsumer = tl.Run
}
@@ -240,9 +240,9 @@ accounts { MESH { jetstream: enabled, users: [
return js, nc.Close, err
}
accepted := func() (map[string]keeper.Planned, error) {
return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}}, nil
return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}, "both": {Module: "both"}}, nil
}
for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo"} {
for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo", "g14_both", "g14_both_tools"} {
if _, err := ajs.CreateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: d, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
@@ -250,41 +250,46 @@ accounts { MESH { jetstream: enabled, users: [
if _, err := ajs.Publish(ctx, "mesh.events.one", []byte("{}")); err != nil {
t.Fatal(err)
}
for _, d := range []string{"novox_m_tools", "ace_q"} {
for _, d := range []string{"novox_m_tools", "ace_q", "g14_both", "g14_both_tools"} {
c, _ := ajs.Consumer(ctx, "EVENTS", d)
if _, err := c.Next(jetstream.FetchMaxWait(2 * time.Second)); err != nil {
t.Fatalf("a delivery to %s: %v", d, err)
}
}
// novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none.
live, _ := ajs.Consumer(ctx, "EVENTS", "novox_m_tools")
held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute))
if err != nil {
t.Fatal(err)
}
defer held.Stop()
for i := 0; ; i++ {
ci, err := live.Info(ctx)
// novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none; both twins of
// g14_both have one each (the third addendum: a durable with pulls waiting is no orphan, its twin current or not).
for _, d := range []string{"novox_m_tools", "g14_both", "g14_both_tools"} {
live, _ := ajs.Consumer(ctx, "EVENTS", d)
held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute))
if err != nil {
t.Fatal(err)
}
if ci.NumWaiting > 0 {
break
}
if i == 1000 {
t.Fatal("the held fetch never shows as a pull waiting")
defer held.Stop()
for i := 0; ; i++ {
ci, err := live.Info(ctx)
if err != nil {
t.Fatal(err)
}
if ci.NumWaiting > 0 {
break
}
if i == 1000 {
t.Fatalf("the held fetch of %s never shows as a pull waiting", d)
}
}
}
handActs = nil
for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) {
for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) {
if tl.Name == "nats_consumer_remove" {
removeConsumer = tl.Run
}
}
for _, row := range []struct{ consumer, refused string }{
{"novox_m_tools", "its twin novox_m on EVENTS is not current"},
{"novox_m_tools", "it is no orphan: 1 pulls waiting on it"},
{"ace_q_tools", "its twin ace_q on EVENTS is not current"},
{"ace_solo", "a module of an accepted manifest"},
{"g14_both", "it is no orphan: 1 pulls waiting on it"},
{"g14_both_tools", "it is no orphan: 1 pulls waiting on it"},
{"novox_m", ""},
} {
_, err := removeConsumer(map[string]any{"stream": "EVENTS", "consumer": row.consumer, "why": "an orphaned durable"})
+29 -11
View File
@@ -279,7 +279,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"},
} {
var run func(map[string]any) (any, error)
for _, tl := range removalTools(bus, said, newForm, accepted, nil) {
for _, tl := range removalTools(bus, said, newForm, accepted, nil, nil) {
if tl.Name == c.tool {
run = tl.Run
}
@@ -295,21 +295,39 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
// twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin.
now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC)
hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour)
day := DefaultTwinCurrentHours * time.Hour
for _, c := range []struct {
name string
info *jetstream.ConsumerInfo
want bool
name string
info *jetstream.ConsumerInfo
within time.Duration
want bool
}{
{"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, true},
{"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, false},
{"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, false},
{"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, false},
{"not read", nil, false},
{"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, true},
{"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, false},
{"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, day, false},
{"delivered two days ago, the setting at 72 hours", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, 72 * time.Hour, true},
{"delivered an hour ago, the setting at 1 hour", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, time.Hour, false},
{"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, day, false},
{"not read", nil, day, false},
} {
if got := current(c.info, now); got != c.want {
if got := current(c.info, now, c.within); got != c.want {
t.Errorf("current, %s: %v, want %v", c.name, got, c.want)
}
}
// H-A-472 third addendum (2): the setting twin-current-hours, in hours; its default 24 when the configuration gives
// none or no positive whole number.
for _, c := range []struct {
raw string
err error
want time.Duration
}{
{"72", nil, 72 * time.Hour}, {"1", nil, time.Hour}, {"", errors.New("no configuration"), 24 * time.Hour},
{"0", nil, 24 * time.Hour}, {"-3", nil, 24 * time.Hour}, {`"x"`, nil, 24 * time.Hour},
} {
if got := twinCurrentWithin(json.RawMessage(c.raw), c.err); got != c.want {
t.Errorf("twin-current-hours %q (%v): %s, want %s", c.raw, c.err, got, c.want)
}
}
for _, c := range []struct{ module, durable, want string }{
{"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""},
} {
@@ -321,7 +339,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
unreachable := func(context.Context) (jetstream.JetStream, func(), error) {
return nil, nil, errors.New("the bus is not reached here")
}
for _, tl := range removalTools(unreachable, said, newForm, accepted, nil) {
for _, tl := range removalTools(unreachable, said, newForm, accepted, nil, nil) {
if tl.Name != "nats_consumer_remove" {
continue
}
+6
View File
@@ -92,6 +92,12 @@
"description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.",
"minimum": 1,
"default": 90
},
"twin-current-hours": {
"type": "number",
"description": "How many hours since a module's durable's twin last delivered it is current (design 47 H2, H-A-472): nats_consumer_remove removes a module's durable with no pulls waiting only while its twin has pulls waiting and a delivery within these hours. The grants part's manifest names the same setting for its orphaned durables. The tool reads it as its configuration at each call.",
"minimum": 1,
"default": 24
}
},
"streams": [