From e0bd0bedc48e776a560e9b9d5730dc3469765941 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:00:11 +0100 Subject: [PATCH] nats: nats_consumer_remove removes an orphaned durable whose twin is current A module that moved into the tool runner left its program's durable _, and one that runs its own program again left __tools: about 110 such orphans the tool refused by name. It now removes a module's durable when its twin on the same stream has pulls waiting and a delivery within the day, and names the twin in the hand act; a module's only durable stays refused. Builds: design 47 H2 H-A-472 (1), H-A-127, H-A-128, G A-340 --- modules/nats/cmd/nats-tools/removal.go | 74 ++++++++++++++++--- modules/nats/cmd/nats-tools/removal_test.go | 78 ++++++++++++++++++++- modules/nats/cmd/nats-tools/tools_test.go | 35 ++++++++- 3 files changed, 174 insertions(+), 13 deletions(-) diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go index c1fd5c86e..2f3270af5 100644 --- a/modules/nats/cmd/nats-tools/removal.go +++ b/modules/nats/cmd/nats-tools/removal.go @@ -349,7 +349,8 @@ func RemoveStream(ctx context.Context, bus Bus, said Said, a map[string]any) (an // recorded as a hand act through mesh-hand-acts, as nats_bucket_expiry's change is. func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModules, record Recorded) []stdio.Tool { removeConsumer := func(a map[string]any) (any, error) { - if err := refuseAcceptedDurable(accepted, arg(a, "consumer")); err != nil { + twin, err := refuseAcceptedDurable(context.Background(), bus, accepted, arg(a, "stream"), arg(a, "consumer"), time.Now()) + if err != nil { return nil, err } out, err := RemoveConsumer(context.Background(), bus, said, a) @@ -358,6 +359,9 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } r := out.(ConsumerRemoved) what := fmt.Sprintf("removed the consumer %s of the stream %s", r.Consumer, r.Stream) + if twin != "" { + what += fmt.Sprintf(", an orphan: its module's twin %s on that stream is current (design 47 H2, H-A-472)", twin) + } if record == nil { return nil, fmt.Errorf("%s WAS done (%s) but the hand-act log cannot be reached here", what, r.Why) } @@ -380,7 +384,7 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu } return []stdio.Tool{ {Name: "nats_consumer_remove", - Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream; any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. a consumer the controller left, such as `controller` of DEAD_LETTER_NOTICES or `controller-merges` of EVENTS (novox/hq ADR 0590 §6), or an orphan on a module's kept-data stream. Refuses a durable named _ or beginning __ for a module of an accepted manifest, on any stream, unless it is an orphan: its twin (_ for __tools, and back) on the same stream is current, with pulls waiting and a delivery within the day, and the hand act names that twin (design 47 H2, H-A-472); any other durable goes. Needs `why`, its warrant, kept in the audit trail and recorded as a hand act through mesh-hand-acts (design 47 H, H-A-127); refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, Run: removeConsumer}, {Name: "nats_stream_remove", @@ -405,24 +409,76 @@ func removalTools(bus Bus, said Said, newForm heldEntries, accepted acceptedModu // state plans (design 47 G §538), never from its state asked, which holds only a module that asks for kept data (a // module keeping none would lose its live durables: H-A-128's silent failure, P3). A set that cannot be read refuses // too: what cannot be told apart is not removed. -func refuseAcceptedDurable(accepted acceptedModules, consumer string) error { +// +// **An orphaned durable goes** (design 47 H2, H-A-472 (1)): a module's durable whose live twin on the same stream exists +// and is current is not refused, and the twin's name is answered, for the hand act to name it. The twins are the +// module's program's durable _ and its tool runner's __tools (design 47 G, A-340): a module +// that moved into the tool runner left the first, one that runs its own program again left the second. A module's +// only durable, any other suffix, and a twin that cannot be read or is not current, stay refused (H-A-127/128). +func refuseAcceptedDurable(ctx context.Context, bus Bus, accepted acceptedModules, stream, consumer string, now time.Time) (string, error) { if accepted == nil { - return nil + return "", nil } node, module, ok := strings.Cut(consumer, "_") if !ok || node == "" || module == "" { - return nil + return "", nil } held, err := accepted() if err != nil { - return fmt.Errorf("whether %s is the durable of a module of an accepted manifest cannot be read (%v), so nothing was removed", consumer, err) + return "", fmt.Errorf("whether %s is the durable of a module of an accepted manifest cannot be read (%v), so nothing was removed", consumer, err) } for m := range held { - if module == m || strings.HasPrefix(module, m+"_") { - return fmt.Errorf("%s is %s's durable on %s, a module of an accepted manifest: it is not removed; nothing was removed", consumer, m, node) + if module != m && !strings.HasPrefix(module, m+"_") { + continue } + refused := fmt.Errorf("%s is %s's durable on %s, a module of an accepted manifest: it is not removed; nothing was removed", consumer, m, node) + twin := twinOf(node, m, module) + if twin == "" || bus == nil || !name.MatchString(stream) { + return "", refused + } + js, closeBus, err := bus(ctx) + if err != nil { + return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + } + sctx, cancel := step(ctx) + info, err := readConsumer(sctx, js, stream, twin) + cancel() + closeBus() + if err != nil { + return "", fmt.Errorf("%v (its twin %s on %s cannot be read: %v)", refused, twin, stream, err) + } + if !current(info, now) { + return "", fmt.Errorf("%v (its twin %s on %s is not current: %d pulls waiting, last delivery %s)", refused, twin, stream, info.NumWaiting, lastDelivery(info)) + } + return twin, nil } - return nil + return "", nil +} + +// twinOf names the twin of a module's durable: __tools for _, and back; "" for any other. +func twinOf(node, module, durable string) string { + switch durable { + case module: + return node + "_" + module + "_tools" + case module + "_tools": + return node + "_" + module + } + return "" +} + +// currentWithin is how recent a twin's last delivery is for it to be current: within the day (H-A-472 (1)). +const currentWithin = 24 * time.Hour + +// current says whether a durable is live: pulls waiting on it and a delivery within the day (H-A-472 (1)). +func current(info *jetstream.ConsumerInfo, now time.Time) bool { + return info != nil && info.NumWaiting > 0 && info.Delivered.Last != nil && now.Sub(*info.Delivered.Last) < currentWithin +} + +func lastDelivery(info *jetstream.ConsumerInfo) string { + if info == nil || info.Delivered.Last == nil { + return "never" + } + return info.Delivered.Last.UTC().Format(time.RFC3339) } // acceptedModules reads the keeper's accepted set: every module it heard as accepted, by name (its state plans, §538). diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go index f924f8f02..2b5d27f1c 100644 --- a/modules/nats/cmd/nats-tools/removal_test.go +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -18,6 +18,8 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + + "nats-tools/internal/keeper" ) // The tools that change the bus, against a server shaped like the mesh's, on the credential the module's @@ -226,6 +228,80 @@ accounts { MESH { jetstream: enabled, users: [ if len(handActs) != 2 || handActs[0].Why != "the controller ended" || !strings.Contains(handActs[1].What, "controller of the stream EVENTS") { t.Fatalf("the hand acts: %+v", handActs) } + // Design 47 H2, H-A-472 (1): an accepted module's durable goes when its twin on the same stream is current (pulls + // waiting, a delivery within the day), the hand act naming the twin; its only durable, and one whose twin is not + // current, stay refused. On the admin's credential: the grant of the old sections names no module's durable. + adminBus := func(context.Context) (jetstream.JetStream, func(), error) { + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "admin")) + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + return js, nc.Close, err + } + accepted := func() (map[string]keeper.Planned, error) { + return map[string]keeper.Planned{"m": {Module: "m"}, "q": {Module: "q"}, "solo": {Module: "solo"}}, nil + } + for _, d := range []string{"novox_m", "novox_m_tools", "ace_q", "ace_q_tools", "ace_solo"} { + if _, err := ajs.CreateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: d, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + if _, err := ajs.Publish(ctx, "mesh.events.one", []byte("{}")); err != nil { + t.Fatal(err) + } + for _, d := range []string{"novox_m_tools", "ace_q"} { + c, _ := ajs.Consumer(ctx, "EVENTS", d) + if _, err := c.Next(jetstream.FetchMaxWait(2 * time.Second)); err != nil { + t.Fatalf("a delivery to %s: %v", d, err) + } + } + // novox_m_tools has a pull waiting (a fetch held open on the admin's connection); ace_q has none. + live, _ := ajs.Consumer(ctx, "EVENTS", "novox_m_tools") + held, err := live.Consume(func(jetstream.Msg) {}, jetstream.PullExpiry(time.Minute)) + if err != nil { + t.Fatal(err) + } + defer held.Stop() + for i := 0; ; i++ { + ci, err := live.Info(ctx) + if err != nil { + t.Fatal(err) + } + if ci.NumWaiting > 0 { + break + } + if i == 1000 { + t.Fatal("the held fetch never shows as a pull waiting") + } + } + handActs = nil + for _, tl := range removalTools(adminBus, say, nil, accepted, func(q meshhandactsclient.Request) error { handActs = append(handActs, q); return nil }) { + if tl.Name == "nats_consumer_remove" { + removeConsumer = tl.Run + } + } + for _, row := range []struct{ consumer, refused string }{ + {"novox_m_tools", "its twin novox_m on EVENTS is not current"}, + {"ace_q_tools", "its twin ace_q on EVENTS is not current"}, + {"ace_solo", "a module of an accepted manifest"}, + {"novox_m", ""}, + } { + _, err := removeConsumer(map[string]any{"stream": "EVENTS", "consumer": row.consumer, "why": "an orphaned durable"}) + _, still := ajs.Consumer(ctx, "EVENTS", row.consumer) + if row.refused == "" { + if err != nil || still == nil { + t.Fatalf("the orphan %s: %v, still there: %v", row.consumer, err, still == nil) + } + continue + } + if err == nil || !strings.Contains(err.Error(), row.refused) || still != nil { + t.Fatalf("%s: %v (still there: %v), want a refusal saying %q", row.consumer, err, still == nil, row.refused) + } + } + if len(handActs) != 1 || !strings.Contains(handActs[0].What, "novox_m of the stream EVENTS") || !strings.Contains(handActs[0].What, "twin novox_m_tools") { + t.Fatalf("the orphan's hand act: %+v", handActs) + } if r, err := RemoveStream(ctx, bus, say, map[string]any{"stream": "KV_mesh-controller_rollouts", "why": "the controller ended"}); err != nil || !r.(StreamRemoved).Removed || r.(StreamRemoved).Messages != 1 { t.Fatalf("the controller's bucket: %+v %v", r, err) } @@ -272,7 +348,7 @@ accounts { MESH { jetstream: enabled, users: [ if k := keys(other); k != "g14.1" { t.Fatalf("another bucket changed: %s", k) } - if strings.Join(said, ",") != "consumer-removed,stream-removed,consumer-removed,consumer-removed,stream-removed,bucket-keys-removed" { + if strings.Join(said, ",") != "consumer-removed,stream-removed,consumer-removed,consumer-removed,consumer-removed,stream-removed,bucket-keys-removed" { t.Fatalf("the audit events said: %v", said) } } diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index ebbbe91ad..90764619d 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -222,11 +222,11 @@ func TestAGrantIsInForceOnlyWhenTheServerReloadedAfterTheListWasWritten(t *testi } // **The tools that change the bus are two, each refuses before it reaches the bus, and every other tool reads** -// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them. +// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them, but to read an accepted module's durable's +// twin (H-A-472), and a twin that cannot be read refuses. func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { bus := func(context.Context) (jetstream.JetStream, func(), error) { - t.Fatal("the bus was opened for a call that must be refused") - return nil, nil, nil + return nil, nil, errors.New("the bus is opened here only to read a twin, and cannot be") } said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil } newForm := func() (map[string]keeper.Asked, error) { @@ -274,6 +274,9 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { // H-A-128: an accepted module that asks for no kept data is refused too, its durables and its suffixed ones. {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet", "why": "x"}, "a module of an accepted manifest"}, {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "a module of an accepted manifest"}, + // H-A-472 (1): a twin that cannot be read keeps the durable refused, saying so. + {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_m", "why": "x"}, "its twin novox_m_tools on EVENTS cannot be read"}, + {"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"}, } { var run func(map[string]any) (any, error) for _, tl := range removalTools(bus, said, newForm, accepted, nil) { @@ -288,6 +291,32 @@ func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { t.Errorf("%s %v: %v, want a refusal saying %q", c.tool, c.args, err, c.want) } } + // H-A-472 (1): a module's durable whose twin (_ and __tools, A-340) is current goes; the + // twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin. + now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC) + hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour) + for _, c := range []struct { + name string + info *jetstream.ConsumerInfo + want bool + }{ + {"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, true}, + {"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, false}, + {"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, false}, + {"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, false}, + {"not read", nil, false}, + } { + if got := current(c.info, now); got != c.want { + t.Errorf("current, %s: %v, want %v", c.name, got, c.want) + } + } + for _, c := range []struct{ module, durable, want string }{ + {"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""}, + } { + if got := twinOf("novox", c.module, c.durable); got != c.want { + t.Errorf("the twin of novox_%s: %q, want %q", c.durable, got, c.want) + } + } // An orphan on a module's kept-data stream is not refused by its stream (H-A-126): the call reaches the bus. unreachable := func(context.Context) (jetstream.JetStream, func(), error) { return nil, nil, errors.New("the bus is not reached here")