diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index ac2774ac3..fc35315a5 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -191,7 +191,7 @@ func allTools() []stdio.Tool { defer cancel() return monitor(ctx, heldJsz) } - out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat)...) + out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat, bundleTwinCurrent)...) out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...) out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index a6330a88a..4c0790563 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -346,6 +346,11 @@ func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) { if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 { t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config) } + // Design 47 H2, H-A-472 third addendum (2): twin-current-hours, the bound of a current twin, defaults to 24, the field + // name the grants part's manifest names too. + if m.Config["twin-current-hours"].Default != DefaultTwinCurrentHours || DefaultTwinCurrentHours != 24 { + t.Errorf("config %v: twin-current-hours defaults to 24", m.Config) + } found := false for _, r := range m.Resources { if r.ID == "keeper-settings" { diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go index 2f3270af5..df6c982dc 100644 --- a/modules/nats/cmd/nats-tools/removal.go +++ b/modules/nats/cmd/nats-tools/removal.go @@ -13,6 +13,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "regexp" @@ -347,9 +348,13 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an // every module it heard as accepted, its state plans, one key per module of a watched branch's newest revision whether // it asks for kept data or not, design 47 G §538) is refused, on any stream; every other durable is an orphan and goes, `why` its warrant, at once, and its removal is // recorded as a hand act through mesh-hand-acts, as nats_bucket_expiry's change is. -func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded) []stdio.Tool { +func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded, twinWithin func() time.Duration) []stdio.Tool { removeConsumer := func(a map[string]any) (any, error) { - twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now()) + within := DefaultTwinCurrentHours * time.Hour + if twinWithin != nil { + within = twinWithin() + } + twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now(), within) if err != nil { return nil, err } @@ -384,7 +389,7 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } return []stdio.Tool{ {Name: "nats_consumer_remove", - Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the day, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: no pulls waiting on it while its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the setting twin-current-hours (24 by default), both read at the call, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, Run: removeConsumer}, {Name: "nats_stream_remove", @@ -410,12 +415,14 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu // module keeping none would lose its live durables: H-A-128's silent failure, P3). A set that cannot be read refuses // too: what cannot be told apart is not removed. // -// **An orphaned durable goes** (design 47 H2, H-A-472 (1)): a module's durable whose live twin on the same stream exists -// and is current is not refused, and the twin's name is answered, for the hand act to name it. The twins are the +// **An orphaned durable goes** (design 47 H2, H-A-472 (1) and its third addendum): a module's durable that is itself an +// orphan (no pulls waiting on it) while its live twin on the same stream exists and is current is not refused, and the +// twin's name is answered, for the hand act to name it; both are read from the bus at the call. The twins are the // module's program's durable _ and its tool runner's __tools (design 47 G, A-340): a module // that moved into the tool runner left the first, one that runs its own program again left the second. A module's -// only durable, any other suffix, and a twin that cannot be read or is not current, stay refused (H-A-127/128). -func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time) (string, error) { +// only durable, any other suffix, a durable with pulls waiting, and a durable or twin that cannot be read or a twin +// that is not current, stay refused (H-A-127/128). within is the bound of "current", the setting twin-current-hours. +func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time, within time.Duration) (string, error) { if accepted == nil { return "", nil } @@ -438,17 +445,26 @@ func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModule } js, closeBus, err := bus(ctx) if err != nil { - return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + return "", fmt.Errorf("%v (it and its twin %s on %s cannot be read: %v)", refused, twin, stream, err) } + defer closeBus() sctx, cancel := step(ctx) + self, err := readConsumer(sctx, js, stream, consumer) + cancel() + if err != nil { + return "", fmt.Errorf("%v (it cannot be read on %s: %v)", refused, stream, err) + } + if self.NumWaiting > 0 { + return "", fmt.Errorf("%v (it is no orphan: %d pulls waiting on it)", refused, self.NumWaiting) + } + sctx, cancel = step(ctx) info, err := readConsumer(sctx, js, stream, twin) cancel() - closeBus() if err != nil { return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) } - if !current(info, now) { - return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info)) + if !current(info, now, within) { + return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s, the bound %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info), within) } return twin, nil } @@ -466,12 +482,30 @@ func twinOf(node, module, durable string) string { return "" } -// currentWithin is how recent a twin's last delivery is for it to be current: within the day (H-A-472 (1)). -const currentWithin = 24 * time.Hour +// DefaultTwinCurrentHours is the setting twin-current-hours's default (design 47 H2, H-A-472 third addendum): how recent a +// twin's last delivery is for it to be current, within the day; the grants part reads the same field name of its own +// manifest for its orphaned durables. +const DefaultTwinCurrentHours = 24 -// current says whether a durable is live: pulls waiting on it and a delivery within the day (H-A-472 (1)). -func current(info *jetstream.ConsumerInfo, now time.Time) bool { - return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < currentWithin +// twinCurrentWithin is the setting twin-current-hours as the configuration answers it, in hours, as a bound: its default +// when the configuration gives none, or a value that is not a positive whole number. +func twinCurrentWithin(raw json.RawMessage, err error) time.Duration { + var n int + if err != nil || json.Unmarshal(raw, &n) != nil || n < 1 { + n = DefaultTwinCurrentHours + } + return time.Duration(n) * time.Hour +} + +// bundleTwinCurrent is the setting twin-current-hours as this bundle's configuration holds it, read at each call +// (never while the tools are listed: see bundleKeepDays). +func bundleTwinCurrent() time.Duration { + return twinCurrentWithin(stdio.Config("twin-current-hours")) +} + +// current says whether a durable is live: pulls waiting on it and a delivery within the bound (H-A-472 (1)). +func current(info *jetstream.ConsumerInfo, now time.Time, within time.Duration) bool { + return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < within } func lastDelivery(info *jetstream.ConsumerInfo) string { diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go index 2b5d27f1c..257431722 100644 --- a/modules/nats/cmd/nats-tools/removal_test.go +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -197,7 +197,7 @@ accounts { MESH { jetstream: enabled, users: [ // Through the tool, `why` its warrant: each removal is recorded as a hand act (design 47 H, H-A-127). var handActs []meshhandactsclient.Request var removeConsumer func(map[string]any) (any, error) - for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) { if tl.Name == "nats_consumer_remove" { removeConsumer = tl.Run } @@ -240,9 +240,9 @@ accounts { MESH { jetstream: enabled, users: [ return js, nc.Close, err } accepted := func() (map[string]keeper.Planned, error) { - return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}}, nil + return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}, "both": {Module: "both"}}, nil } - for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo"} { + for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo", "g14_both", "g14_both_tools"} { if _, err := ajs.CreateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: d, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } @@ -250,41 +250,46 @@ accounts { MESH { jetstream: enabled, users: [ if _, err := ajs.Publish(ctx, "mesh.events.one", []byte("{}")); err != nil { t.Fatal(err) } - for _, d := range []string{"novox_m_tools", "ace_q"} { + for _, d := range []string{"novox_m_tools", "ace_q", "g14_both", "g14_both_tools"} { c, _ := ajs.Consumer(ctx, "EVENTS", d) if _, err := c.Next(jetstream.FetchMaxWait(2 * time.Second)); err != nil { t.Fatalf("a delivery to %s: %v", d, err) } } - // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none. - live, _ := ajs.Consumer(ctx, "EVENTS", "novox_m_tools") - held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) - if err != nil { - t.Fatal(err) - } - defer held.Stop() - for i := 0; ; i++ { - ci, err := live.Info(ctx) + // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none; both twins of + // g14_both have one each (the third addendum: a durable with pulls waiting is no orphan, its twin current or not). + for _, d := range []string{"novox_m_tools", "g14_both", "g14_both_tools"} { + live, _ := ajs.Consumer(ctx, "EVENTS", d) + held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) if err != nil { t.Fatal(err) } - if ci.NumWaiting > 0 { - break - } - if i == 1000 { - t.Fatal("the held fetch never shows as a pull waiting") + defer held.Stop() + for i := 0; ; i++ { + ci, err := live.Info(ctx) + if err != nil { + t.Fatal(err) + } + if ci.NumWaiting > 0 { + break + } + if i == 1000 { + t.Fatalf("the held fetch of %s never shows as a pull waiting", d) + } } } handActs = nil - for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) { if tl.Name == "nats_consumer_remove" { removeConsumer = tl.Run } } for _, row := range []struct{ consumer, refused string }{ - {"novox_m_tools", "its twin novox_m on EVENTS is not current"}, + {"novox_m_tools", "it is no orphan: 1 pulls waiting on it"}, {"ace_q_tools", "its twin ace_q on EVENTS is not current"}, {"ace_solo", "a module of an accepted manifest"}, + {"g14_both", "it is no orphan: 1 pulls waiting on it"}, + {"g14_both_tools", "it is no orphan: 1 pulls waiting on it"}, {"novox_m", ""}, } { _, err := removeConsumer(map[string]any{"stream": "EVENTS", "consumer": row.consumer, "why": "an orphaned durable"}) diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index 90764619d..1bc0b8eb4 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -279,7 +279,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"}, } { var run func(map[string]any) (any, error) - for _, tl := range removalTools(bus, said, newForm, accepted, nil) { + for _, tl := range removalTools(bus, said, newForm, accepted, nil, nil) { if tl.Name == c.tool { run = tl.Run } @@ -295,21 +295,39 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { // twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin. now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC) hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour) + day := DefaultTwinCurrentHours * time.Hour for _, c := range []struct { - name string - info *jetstream.ConsumerInfo - want bool + name string + info *jetstream.ConsumerInfo + within time.Duration + want bool }{ - {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, true}, - {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, false}, - {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, false}, - {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, false}, - {"not read", nil, false}, + {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, true}, + {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, false}, + {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, day, false}, + {"delivered two days ago, the setting at 72 hours", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, 72 * time.Hour, true}, + {"delivered an hour ago, the setting at 1 hour", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, time.Hour, false}, + {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, day, false}, + {"not read", nil, day, false}, } { - if got := current(c.info, now); got != c.want { + if got := current(c.info, now, c.within); got != c.want { t.Errorf("current, %s: %v, want %v", c.name, got, c.want) } } + // H-A-472 third addendum (2): the setting twin-current-hours, in hours; its default 24 when the configuration gives + // none or no positive whole number. + for _, c := range []struct { + raw string + err error + want time.Duration + }{ + {"72", nil, 72 * time.Hour}, {"1", nil, time.Hour}, {"", errors.New("no configuration"), 24 * time.Hour}, + {"0", nil, 24 * time.Hour}, {"-3", nil, 24 * time.Hour}, {`"x"`, nil, 24 * time.Hour}, + } { + if got := twinCurrentWithin(json.RawMessage(c.raw), c.err); got != c.want { + t.Errorf("twin-current-hours %q (%v): %s, want %s", c.raw, c.err, got, c.want) + } + } for _, c := range []struct{ module, durable, want string }{ {"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""}, } { @@ -321,7 +339,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { unreachable := func(context.Context) (jetstream.JetStream, func(), error) { return nil, nil, errors.New("the bus is not reached here") } - for _, tl := range removalTools(unreachable, said, newForm, accepted, nil) { + for _, tl := range removalTools(unreachable, said, newForm, accepted, nil, nil) { if tl.Name != "nats_consumer_remove" { continue } diff --git a/modules/nats/module.json b/modules/nats/module.json index bcc65be90..57b75d50d 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -92,6 +92,12 @@ "description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.", "minimum": 1, "default": 90 + }, + "twin-current-hours": { + "type": "number", + "description": "How many hours since a module's durable's twin last delivered it is current (design 47 H2, H-A-472): nats_consumer_remove removes a module's durable with no pulls waiting only while its twin has pulls waiting and a delivery within these hours. The grants part's manifest names the same setting for its orphaned durables. The tool reads it as its configuration at each call.", + "minimum": 1, + "default": 24 } }, "streams": [