From e0bd0bedc48e776a560e9b9d5730dc3469765941 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:00:11 +0100 Subject: [PATCH 1/2] nats: nats_consumer_remove removes an orphaned durable whose twin is current A module that moved into the tool runner left its program's durable _, and one that runs its own program again left __tools: about 110 such orphans the tool refused by name. It now removes a module's durable when its twin on the same stream has pulls waiting and a delivery within the day, and names the twin in the hand act; a module's only durable stays refused. Builds: design 47 H2 H-A-472 (1), H-A-127, H-A-128, G A-340 --- modules/nats/cmd/nats-tools/removal.go | 74 ++++++++++++++++--- modules/nats/cmd/nats-tools/removal_test.go | 78 ++++++++++++++++++++- modules/nats/cmd/nats-tools/tools_test.go | 35 ++++++++- 3 files changed, 174 insertions(+), 13 deletions(-) diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go index c1fd5c86e..2f3270af5 100644 --- a/modules/nats/cmd/nats-tools/removal.go +++ b/modules/nats/cmd/nats-tools/removal.go @@ -349,7 +349,8 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an // recorded as a hand act through mesh-hand-acts, as nats_bucket_expiry's change is. func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded) []stdio.Tool { removeConsumer := func(a map[string]any) (any, error) { - if err := refuseAcceptedDurable(accepted, arg(a, "consumer")); err != nil { + twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now()) + if err != nil { return nil, err } out, err := RemoveConsumer(context.Background(), bus, said, a) @@ -358,6 +359,9 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } r := out.(ConsumerRemoved) what := fmt.Sprintf("removed the consumer %s of the stream %s", r.Consumer, r.Stream) + if twin != "" { + what += fmt.Sprintf(", an orphan: its module's twin %s on that stream is current (design 47 H2, H-A-472)", twin) + } if record == nil { return nil, fmt.Errorf("%s WAS done (%s) but the hand-act log cannot be reached here", what, r.Why) } @@ -380,7 +384,7 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } return []stdio.Tool{ {Name: "nats_consumer_remove", - Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream; any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the day, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, Run: removeConsumer}, {Name: "nats_stream_remove", @@ -405,24 +409,76 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu // state plans (design 47 G §538), never from its state asked, which holds only a module that asks for kept data (a // module keeping none would lose its live durables: H-A-128's silent failure, P3). A set that cannot be read refuses // too: what cannot be told apart is not removed. -func refuseAcceptedDurable(accepted acceptedModules, consumer string) error { +// +// **An orphaned durable goes** (design 47 H2, H-A-472 (1)): a module's durable whose live twin on the same stream exists +// and is current is not refused, and the twin's name is answered, for the hand act to name it. The twins are the +// module's program's durable _ and its tool runner's __tools (design 47 G, A-340): a module +// that moved into the tool runner left the first, one that runs its own program again left the second. A module's +// only durable, any other suffix, and a twin that cannot be read or is not current, stay refused (H-A-127/128). +func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time) (string, error) { if accepted == nil { - return nil + return "", nil } node, module, ok := strings.Cut(consumer, "_") if !ok || node == "" || module == "" { - return nil + return "", nil } held, err := accepted() if err != nil { - return fmt.Errorf("whether %s is the durable of a module of an accepted manifest cannot be read (%v), so nothing was removed", consumer, err) + return "", fmt.Errorf("whether %s is the durable of a module of an accepted manifest cannot be read (%v), so nothing was removed", consumer, err) } for m := range held { - if module == m || strings.HasPrefix(module, m+"_") { - return fmt.Errorf("%s is %s's durable on %s, a module of an accepted manifest: it is not removed; nothing was removed", consumer, m, node) + if module != m && !strings.HasPrefix(module, m+"_") { + continue } + refused := fmt.Errorf("%s is %s's durable on %s, a module of an accepted manifest: it is not removed; nothing was removed", consumer, m, node) + twin := twinOf(node, m, module) + if twin == "" || bus == nil || !name.MatchString(stream) { + return "", refused + } + js, closeBus, err := bus(ctx) + if err != nil { + return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + } + sctx, cancel := step(ctx) + info, err := readConsumer(sctx, js, stream, twin) + cancel() + closeBus() + if err != nil { + return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + } + if !current(info, now) { + return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info)) + } + return twin, nil } - return nil + return "", nil +} + +// twinOf names the twin of a module's durable: __tools for _, and back; "" for any other. +func twinOf(node, module, durable string) string { + switch durable { + case module: + return node + "_" + module + "_tools" + case module + "_tools": + return node + "_" + module + } + return "" +} + +// currentWithin is how recent a twin's last delivery is for it to be current: within the day (H-A-472 (1)). +const currentWithin = 24 * time.Hour + +// current says whether a durable is live: pulls waiting on it and a delivery within the day (H-A-472 (1)). +func current(info *jetstream.ConsumerInfo, now time.Time) bool { + return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < currentWithin +} + +func lastDelivery(info *jetstream.ConsumerInfo) string { + if info == nil || info.Delivered.Last == nil { + return "never" + } + return info.Delivered.Last.UTC().Format(time.RFC3339) } // acceptedModules reads the keeper's accepted set: every module it heard as accepted, by name (its state plans, §538). diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go index f924f8f02..2b5d27f1c 100644 --- a/modules/nats/cmd/nats-tools/removal_test.go +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -18,6 +18,8 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + + "nats-tools/internal/keeper" ) // The tools that change the bus, against a server shaped like the mesh's, on the credential the module's @@ -226,6 +228,80 @@ accounts { MESH { jetstream: enabled, users: [ if len(handActs) != 2 || handActs[0].Why != "the controller ended" || !strings.Contains(handActs[1].What, "controller of the stream EVENTS") { t.Fatalf("the hand acts: %+v", handActs) } + // Design 47 H2, H-A-472 (1): an accepted module's durable goes when its twin on the same stream is current (pulls + // waiting, a delivery within the day), the hand act naming the twin; its only durable, and one whose twin is not + // current, stay refused. On the admin's credential: the grant of the old sections names no module's durable. + adminBus := func(context.Context) (jetstream.JetStream, func(), error) { + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "admin")) + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + return js, nc.Close, err + } + accepted := func() (map[string]keeper.Planned, error) { + return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}}, nil + } + for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo"} { + if _, err := ajs.CreateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: d, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + if _, err := ajs.Publish(ctx, "mesh.events.one", []byte("{}")); err != nil { + t.Fatal(err) + } + for _, d := range []string{"novox_m_tools", "ace_q"} { + c, _ := ajs.Consumer(ctx, "EVENTS", d) + if _, err := c.Next(jetstream.FetchMaxWait(2 * time.Second)); err != nil { + t.Fatalf("a delivery to %s: %v", d, err) + } + } + // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none. + live, _ := ajs.Consumer(ctx, "EVENTS", "novox_m_tools") + held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) + if err != nil { + t.Fatal(err) + } + defer held.Stop() + for i := 0; ; i++ { + ci, err := live.Info(ctx) + if err != nil { + t.Fatal(err) + } + if ci.NumWaiting > 0 { + break + } + if i == 1000 { + t.Fatal("the held fetch never shows as a pull waiting") + } + } + handActs = nil + for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + if tl.Name == "nats_consumer_remove" { + removeConsumer = tl.Run + } + } + for _, row := range []struct{ consumer, refused string }{ + {"novox_m_tools", "its twin novox_m on EVENTS is not current"}, + {"ace_q_tools", "its twin ace_q on EVENTS is not current"}, + {"ace_solo", "a module of an accepted manifest"}, + {"novox_m", ""}, + } { + _, err := removeConsumer(map[string]any{"stream": "EVENTS", "consumer": row.consumer, "why": "an orphaned durable"}) + _, still := ajs.Consumer(ctx, "EVENTS", row.consumer) + if row.refused == "" { + if err != nil || still == nil { + t.Fatalf("the orphan %s: %v, still there: %v", row.consumer, err, still == nil) + } + continue + } + if err == nil || !strings.Contains(err.Error(), row.refused) || still != nil { + t.Fatalf("%s: %v (still there: %v), want a refusal saying %q", row.consumer, err, still == nil, row.refused) + } + } + if len(handActs) != 1 || !strings.Contains(handActs[0].What, "novox_m of the stream EVENTS") || !strings.Contains(handActs[0].What, "twin novox_m_tools") { + t.Fatalf("the orphan's hand act: %+v", handActs) + } if r, err := RemoveStream(ctx, bus, say, map[string]any{"stream": "KV_mesh-controller_rollouts", "why": "the controller ended"}); err != nil || !r.(StreamRemoved).Removed || r.(StreamRemoved).Messages != 1 { t.Fatalf("the controller's bucket: %+v %v", r, err) } @@ -272,7 +348,7 @@ accounts { MESH { jetstream: enabled, users: [ if k := keys(other); k != "g14.1" { t.Fatalf("another bucket changed: %s", k) } - if strings.Join(said, ",") != "consumer-removed,stream-removed,consumer-removed,consumer-removed,stream-removed,bucket-keys-removed" { + if strings.Join(said, ",") != "consumer-removed,stream-removed,consumer-removed,consumer-removed,consumer-removed,stream-removed,bucket-keys-removed" { t.Fatalf("the audit events said: %v", said) } } diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index ebbbe91ad..90764619d 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -222,11 +222,11 @@ func TestAGrantIsInForceOnlyWhenTheServerReloadedAfterTheListWasWritten(t *testi } // **The tools that change the bus are two, each refuses before it reaches the bus, and every other tool reads** -// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them. +// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them, but to read an accepted module's durable's +// twin (H-A-472), and a twin that cannot be read refuses. func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { bus := func(context.Context) (jetstream.JetStream, func(), error) { - t.Fatal("the bus was opened for a call that must be refused") - return nil, nil, nil + return nil, nil, errors.New("the bus is opened here only to read a twin, and cannot be") } said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil } newForm := func() (map[string]keeper.Asked, error) { @@ -274,6 +274,9 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { // H-A-128: an accepted module that asks for no kept data is refused too, its durables and its suffixed ones. {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet", "why": "x"}, "a module of an accepted manifest"}, {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "a module of an accepted manifest"}, + // H-A-472 (1): a twin that cannot be read keeps the durable refused, saying so. + {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_m", "why": "x"}, "its twin novox_m_tools on EVENTS cannot be read"}, + {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"}, } { var run func(map[string]any) (any, error) for _, tl := range removalTools(bus, said, newForm, accepted, nil) { @@ -288,6 +291,32 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { t.Errorf("%s %v: %v, want a refusal saying %q", c.tool, c.args, err, c.want) } } + // H-A-472 (1): a module's durable whose twin (_ and __tools, A-340) is current goes; the + // twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin. + now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC) + hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour) + for _, c := range []struct { + name string + info *jetstream.ConsumerInfo + want bool + }{ + {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, true}, + {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, false}, + {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, false}, + {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, false}, + {"not read", nil, false}, + } { + if got := current(c.info, now); got != c.want { + t.Errorf("current, %s: %v, want %v", c.name, got, c.want) + } + } + for _, c := range []struct{ module, durable, want string }{ + {"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""}, + } { + if got := twinOf("novox", c.module, c.durable); got != c.want { + t.Errorf("the twin of novox_%s: %q, want %q", c.durable, got, c.want) + } + } // An orphan on a module's kept-data stream is not refused by its stream (H-A-126): the call reaches the bus. unreachable := func(context.Context) (jetstream.JetStream, func(), error) { return nil, nil, errors.New("the bus is not reached here") -- 2.54.0 From 3443760205ab2902ece4303b746b49966f325fab Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:33:01 +0100 Subject: [PATCH 2/2] nats: nats_consumer_remove removes only a durable that is itself an orphan Review round 1: a durable whose twin is current was removed without reading the durable itself, so of the five live pairs where both twins have pulls waiting either one would have gone. The tool now reads the named durable too and refuses it while pulls wait on it, both read from the bus at the call. "Current" is the setting twin-current-hours (24 by default) of the manifest, read from the configuration at each call, the field name the grants part's manifest names too. Builds: design 47 H2 H-A-472 (1) and its third addendum (1), (2); review round 1 findings (1), (2) --- modules/nats/cmd/nats-tools/main.go | 2 +- modules/nats/cmd/nats-tools/manifest_test.go | 5 ++ modules/nats/cmd/nats-tools/removal.go | 66 +++++++++++++++----- modules/nats/cmd/nats-tools/removal_test.go | 45 +++++++------ modules/nats/cmd/nats-tools/tools_test.go | 40 ++++++++---- modules/nats/module.json | 6 ++ 6 files changed, 116 insertions(+), 48 deletions(-) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index ac2774ac3..fc35315a5 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -191,7 +191,7 @@ func allTools() []stdio.Tool { defer cancel() return monitor(ctx, heldJsz) } - out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat)...) + out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState, PlansInState, RecordedBySeat, bundleTwinCurrent)...) out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...) out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index a6330a88a..4c0790563 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -346,6 +346,11 @@ func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) { if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 { t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config) } + // Design 47 H2, H-A-472 third addendum (2): twin-current-hours, the bound of a current twin, defaults to 24, the field + // name the grants part's manifest names too. + if m.Config["twin-current-hours"].Default != DefaultTwinCurrentHours || DefaultTwinCurrentHours != 24 { + t.Errorf("config %v: twin-current-hours defaults to 24", m.Config) + } found := false for _, r := range m.Resources { if r.ID == "keeper-settings" { diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go index 2f3270af5..df6c982dc 100644 --- a/modules/nats/cmd/nats-tools/removal.go +++ b/modules/nats/cmd/nats-tools/removal.go @@ -13,6 +13,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "regexp" @@ -347,9 +348,13 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an // every module it heard as accepted, its state plans, one key per module of a watched branch's newest revision whether // it asks for kept data or not, design 47 G §538) is refused, on any stream; every other durable is an orphan and goes, `why` its warrant, at once, and its removal is // recorded as a hand act through mesh-hand-acts, as nats_bucket_expiry's change is. -func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded) []stdio.Tool { +func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded, twinWithin func() time.Duration) []stdio.Tool { removeConsumer := func(a map[string]any) (any, error) { - twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now()) + within := DefaultTwinCurrentHours * time.Hour + if twinWithin != nil { + within = twinWithin() + } + twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now(), within) if err != nil { return nil, err } @@ -384,7 +389,7 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } return []stdio.Tool{ {Name: "nats_consumer_remove", - Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the day, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: no pulls waiting on it while its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the setting twin-current-hours (24 by default), both read at the call, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, Run: removeConsumer}, {Name: "nats_stream_remove", @@ -410,12 +415,14 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu // module keeping none would lose its live durables: H-A-128's silent failure, P3). A set that cannot be read refuses // too: what cannot be told apart is not removed. // -// **An orphaned durable goes** (design 47 H2, H-A-472 (1)): a module's durable whose live twin on the same stream exists -// and is current is not refused, and the twin's name is answered, for the hand act to name it. The twins are the +// **An orphaned durable goes** (design 47 H2, H-A-472 (1) and its third addendum): a module's durable that is itself an +// orphan (no pulls waiting on it) while its live twin on the same stream exists and is current is not refused, and the +// twin's name is answered, for the hand act to name it; both are read from the bus at the call. The twins are the // module's program's durable _ and its tool runner's __tools (design 47 G, A-340): a module // that moved into the tool runner left the first, one that runs its own program again left the second. A module's -// only durable, any other suffix, and a twin that cannot be read or is not current, stay refused (H-A-127/128). -func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time) (string, error) { +// only durable, any other suffix, a durable with pulls waiting, and a durable or twin that cannot be read or a twin +// that is not current, stay refused (H-A-127/128). within is the bound of "current", the setting twin-current-hours. +func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time, within time.Duration) (string, error) { if accepted == nil { return "", nil } @@ -438,17 +445,26 @@ func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModule } js, closeBus, err := bus(ctx) if err != nil { - return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + return "", fmt.Errorf("%v (it and its twin %s on %s cannot be read: %v)", refused, twin, stream, err) } + defer closeBus() sctx, cancel := step(ctx) + self, err := readConsumer(sctx, js, stream, consumer) + cancel() + if err != nil { + return "", fmt.Errorf("%v (it cannot be read on %s: %v)", refused, stream, err) + } + if self.NumWaiting > 0 { + return "", fmt.Errorf("%v (it is no orphan: %d pulls waiting on it)", refused, self.NumWaiting) + } + sctx, cancel = step(ctx) info, err := readConsumer(sctx, js, stream, twin) cancel() - closeBus() if err != nil { return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) } - if !current(info, now) { - return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info)) + if !current(info, now, within) { + return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s, the bound %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info), within) } return twin, nil } @@ -466,12 +482,30 @@ func twinOf(node, module, durable string) string { return "" } -// currentWithin is how recent a twin's last delivery is for it to be current: within the day (H-A-472 (1)). -const currentWithin = 24 * time.Hour +// DefaultTwinCurrentHours is the setting twin-current-hours's default (design 47 H2, H-A-472 third addendum): how recent a +// twin's last delivery is for it to be current, within the day; the grants part reads the same field name of its own +// manifest for its orphaned durables. +const DefaultTwinCurrentHours = 24 -// current says whether a durable is live: pulls waiting on it and a delivery within the day (H-A-472 (1)). -func current(info *jetstream.ConsumerInfo, now time.Time) bool { - return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < currentWithin +// twinCurrentWithin is the setting twin-current-hours as the configuration answers it, in hours, as a bound: its default +// when the configuration gives none, or a value that is not a positive whole number. +func twinCurrentWithin(raw json.RawMessage, err error) time.Duration { + var n int + if err != nil || json.Unmarshal(raw, &n) != nil || n < 1 { + n = DefaultTwinCurrentHours + } + return time.Duration(n) * time.Hour +} + +// bundleTwinCurrent is the setting twin-current-hours as this bundle's configuration holds it, read at each call +// (never while the tools are listed: see bundleKeepDays). +func bundleTwinCurrent() time.Duration { + return twinCurrentWithin(stdio.Config("twin-current-hours")) +} + +// current says whether a durable is live: pulls waiting on it and a delivery within the bound (H-A-472 (1)). +func current(info *jetstream.ConsumerInfo, now time.Time, within time.Duration) bool { + return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < within } func lastDelivery(info *jetstream.ConsumerInfo) string { diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go index 2b5d27f1c..257431722 100644 --- a/modules/nats/cmd/nats-tools/removal_test.go +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -197,7 +197,7 @@ accounts { MESH { jetstream: enabled, users: [ // Through the tool, `why` its warrant: each removal is recorded as a hand act (design 47 H, H-A-127). var handActs []meshhandactsclient.Request var removeConsumer func(map[string]any) (any, error) - for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + for _, tl := range removalTools(bus, say, nil, nil, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) { if tl.Name == "nats_consumer_remove" { removeConsumer = tl.Run } @@ -240,9 +240,9 @@ accounts { MESH { jetstream: enabled, users: [ return js, nc.Close, err } accepted := func() (map[string]keeper.Planned, error) { - return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}}, nil + return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}, "both": {Module: "both"}}, nil } - for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo"} { + for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo", "g14_both", "g14_both_tools"} { if _, err := ajs.CreateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: d, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } @@ -250,41 +250,46 @@ accounts { MESH { jetstream: enabled, users: [ if _, err := ajs.Publish(ctx, "mesh.events.one", []byte("{}")); err != nil { t.Fatal(err) } - for _, d := range []string{"novox_m_tools", "ace_q"} { + for _, d := range []string{"novox_m_tools", "ace_q", "g14_both", "g14_both_tools"} { c, _ := ajs.Consumer(ctx, "EVENTS", d) if _, err := c.Next(jetstream.FetchMaxWait(2 * time.Second)); err != nil { t.Fatalf("a delivery to %s: %v", d, err) } } - // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none. - live, _ := ajs.Consumer(ctx, "EVENTS", "novox_m_tools") - held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) - if err != nil { - t.Fatal(err) - } - defer held.Stop() - for i := 0; ; i++ { - ci, err := live.Info(ctx) + // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none; both twins of + // g14_both have one each (the third addendum: a durable with pulls waiting is no orphan, its twin current or not). + for _, d := range []string{"novox_m_tools", "g14_both", "g14_both_tools"} { + live, _ := ajs.Consumer(ctx, "EVENTS", d) + held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) if err != nil { t.Fatal(err) } - if ci.NumWaiting > 0 { - break - } - if i == 1000 { - t.Fatal("the held fetch never shows as a pull waiting") + defer held.Stop() + for i := 0; ; i++ { + ci, err := live.Info(ctx) + if err != nil { + t.Fatal(err) + } + if ci.NumWaiting > 0 { + break + } + if i == 1000 { + t.Fatalf("the held fetch of %s never shows as a pull waiting", d) + } } } handActs = nil - for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }, nil) { if tl.Name == "nats_consumer_remove" { removeConsumer = tl.Run } } for _, row := range []struct{ consumer, refused string }{ - {"novox_m_tools", "its twin novox_m on EVENTS is not current"}, + {"novox_m_tools", "it is no orphan: 1 pulls waiting on it"}, {"ace_q_tools", "its twin ace_q on EVENTS is not current"}, {"ace_solo", "a module of an accepted manifest"}, + {"g14_both", "it is no orphan: 1 pulls waiting on it"}, + {"g14_both_tools", "it is no orphan: 1 pulls waiting on it"}, {"novox_m", ""}, } { _, err := removeConsumer(map[string]any{"stream": "EVENTS", "consumer": row.consumer, "why": "an orphaned durable"}) diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index 90764619d..1bc0b8eb4 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -279,7 +279,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"}, } { var run func(map[string]any) (any, error) - for _, tl := range removalTools(bus, said, newForm, accepted, nil) { + for _, tl := range removalTools(bus, said, newForm, accepted, nil, nil) { if tl.Name == c.tool { run = tl.Run } @@ -295,21 +295,39 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { // twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin. now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC) hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour) + day := DefaultTwinCurrentHours * time.Hour for _, c := range []struct { - name string - info *jetstream.ConsumerInfo - want bool + name string + info *jetstream.ConsumerInfo + within time.Duration + want bool }{ - {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, true}, - {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, false}, - {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, false}, - {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, false}, - {"not read", nil, false}, + {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, true}, + {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, false}, + {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, day, false}, + {"delivered two days ago, the setting at 72 hours", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, 72 * time.Hour, true}, + {"delivered an hour ago, the setting at 1 hour", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, time.Hour, false}, + {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, day, false}, + {"not read", nil, day, false}, } { - if got := current(c.info, now); got != c.want { + if got := current(c.info, now, c.within); got != c.want { t.Errorf("current, %s: %v, want %v", c.name, got, c.want) } } + // H-A-472 third addendum (2): the setting twin-current-hours, in hours; its default 24 when the configuration gives + // none or no positive whole number. + for _, c := range []struct { + raw string + err error + want time.Duration + }{ + {"72", nil, 72 * time.Hour}, {"1", nil, time.Hour}, {"", errors.New("no configuration"), 24 * time.Hour}, + {"0", nil, 24 * time.Hour}, {"-3", nil, 24 * time.Hour}, {`"x"`, nil, 24 * time.Hour}, + } { + if got := twinCurrentWithin(json.RawMessage(c.raw), c.err); got != c.want { + t.Errorf("twin-current-hours %q (%v): %s, want %s", c.raw, c.err, got, c.want) + } + } for _, c := range []struct{ module, durable, want string }{ {"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""}, } { @@ -321,7 +339,7 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { unreachable := func(context.Context) (jetstream.JetStream, func(), error) { return nil, nil, errors.New("the bus is not reached here") } - for _, tl := range removalTools(unreachable, said, newForm, accepted, nil) { + for _, tl := range removalTools(unreachable, said, newForm, accepted, nil, nil) { if tl.Name != "nats_consumer_remove" { continue } diff --git a/modules/nats/module.json b/modules/nats/module.json index bcc65be90..57b75d50d 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -92,6 +92,12 @@ "description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.", "minimum": 1, "default": 90 + }, + "twin-current-hours": { + "type": "number", + "description": "How many hours since a module's durable's twin last delivered it is current (design 47 H2, H-A-472): nats_consumer_remove removes a module's durable with no pulls waiting only while its twin has pulls waiting and a delivery within these hours. The grants part's manifest names the same setting for its orphaned durables. The tool reads it as its configuration at each call.", + "minimum": 1, + "default": 24 } }, "streams": [ -- 2.54.0