Files
mesh-catalog/modules/nats/cmd/nats-tools/tools_test.go
T
jschoubben 3443760205 nats: nats_consumer_remove removes only a durable that is itself an orphan
Review round 1: a durable whose twin is current was removed without
reading the durable itself, so of the five live pairs where both twins
have pulls waiting either one would have gone. The tool now reads the
named durable too and refuses it while pulls wait on it, both read from
the bus at the call. "Current" is the setting twin-current-hours (24 by
default) of the manifest, read from the configuration at each call, the
field name the grants part's manifest names too.

Builds: design 47 H2 H-A-472 (1) and its third addendum (1), (2); review round 1 findings (1), (2)
2026-11-22 14:33:01 +01:00

371 lines
19 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"nats-tools/internal/keeper"
)
const connz = `{"connections":[
{"cid":1,"ip":"192.0.2.1","name":"mesh-host/anchor","authorized_user":"node.anchor","out_bytes":10,"in_bytes":5,"pending_bytes":0,"subscriptions":2,"in_msgs":3,"out_msgs":4},
{"cid":2,"ip":"192.0.2.2","name":"laptop.node-tools","authorized_user":"laptop.node-tools","out_bytes":900,"in_bytes":50,"pending_bytes":7,"subscriptions":300,"in_msgs":30,"out_msgs":40,
"subscriptions_list_detail":[{"subject":"a","msgs":1},{"subject":"b","qgroup":"q","msgs":60},{"subject":"c","msgs":5},
{"subject":"d","msgs":40},{"subject":"e","msgs":3},{"subject":"f","msgs":2}]}]}`
const subsz = `{"num_subscriptions":3,"subscriptions_list":[
{"subject":"$SRV.INFO","msgs":4,"cid":2},
{"subject":"mesh.seat.x.tool.current","qgroup":"","msgs":9,"cid":2},
{"subject":"mesh.node.anchor.declare","msgs":1,"cid":1}]}`
const jsz = `{"streams":2,"consumers":2,"messages":12,"bytes":300,"account_details":[{"stream_detail":[
{"name":"EVENTS","config":{"subjects":["mesh.mod.*.event.>"],"retention":"limits","max_age":604800000000000,"max_msgs_per_subject":10000,"max_bytes":1048576},
"state":{"messages":10,"bytes":200,"num_subjects":3,"consumer_count":2,"last_ts":"2026-10-04T10:00:00Z"},
"consumer_detail":[
{"name":"anchor_audit","config":{"filter_subject":"mesh.mod.*.event.>"},"num_pending":5,"num_ack_pending":1,"num_redelivered":2,"delivered":{"last_active":"2026-10-04T10:00:00Z"}},
{"name":"anchor_quiet","config":{"filter_subjects":["mesh.mod.a.event.b"]},"num_pending":0,"num_ack_pending":0}]},
{"name":"KV_claude-code_servers","config":{"subjects":["$KV.claude-code_servers.>"],"retention":"limits","max_msgs_per_subject":1},
"state":{"messages":2,"bytes":100,"num_subjects":2,"consumer_count":1,"last_ts":"2026-10-04T11:00:00Z"}}]}]}`
const users = `accounts {
MESH {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$10$secret-hash-one", permissions: {
publish: { allow: ["mesh.control.>", "$JS.API.>"] }
subscribe: { allow: ["mesh.control.>", "_INBOX.controller.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "laptop.node-tools", password: "$2a$10$secret-hash-two", permissions: {
publish: { allow: ["$KV.claude-code_servers.>", "mesh.mod.*.tool.>"] }
subscribe: { allow: ["mesh.mod.claude-code.tool.>"] }
} }
]
}
}`
func TestConnectionsAreNamedByTheirUserAndSorted(t *testing.T) {
out, err := Connections([]byte(connz), "", "", 10)
if err != nil {
t.Fatal(err)
}
cs := out["connections"].([]Connection)
if len(cs) != 2 || cs[0].User != "laptop.node-tools" || cs[1].User != "node.anchor" {
t.Fatalf("%+v", cs)
}
// Design 47 H2 §813 (5), H-A-356: a connection's five busiest subscriptions by messages, its queue group named.
if b := cs[0].Busiest; fmt.Sprint(b) != "[{b q 60} {d 40} {c 5} {e 3} {f 2}]" || cs[1].Busiest != nil {
t.Fatalf("busiest %v, %v", b, cs[1].Busiest)
}
narrowed, _ := Connections([]byte(connz), "anchor", "pending", 10)
if n := narrowed["total"].(int); n != 1 {
t.Fatalf("narrowed to %d", n)
}
}
func TestSubscriptionsNameTheirUser(t *testing.T) {
out, err := Subscriptions([]byte(subsz), []byte(connz), "SRV", 10)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(out)
if !strings.Contains(string(raw), `"user":"laptop.node-tools"`) || out["matched"].(int) != 1 {
t.Fatalf("%s", raw)
}
}
func TestStreamsBacklogAndBuckets(t *testing.T) {
all, err := Streams([]byte(jsz), "")
if err != nil || len(all["detail"].([]map[string]any)) != 2 {
t.Fatalf("%v %v", all, err)
}
one, _ := Streams([]byte(jsz), "EVENTS")
row := one["detail"].([]map[string]any)[0]
if row["max_age_hours"].(int64) != 168 || row["max_bytes"].(int64) != 1048576 || len(row["consumer_detail"].([]map[string]any)) != 2 {
t.Fatalf("%v", row)
}
if _, err := Streams([]byte(jsz), "NOPE"); err == nil {
t.Fatal("an unknown stream answered")
}
b, _ := Backlog([]byte(jsz))
if b["consumers_behind"].(int) != 1 {
t.Fatalf("%v", b)
}
k, _ := Buckets([]byte(jsz))
rows := k["buckets"].([]map[string]any)
if len(rows) != 1 || rows[0]["module"] != "claude-code" || rows[0]["state"] != "servers" {
t.Fatalf("%v", rows)
}
}
func TestUsersNeverCarryAPasswordHash(t *testing.T) {
for _, out := range []map[string]any{Users([]byte(users), ""), Users([]byte(users), "laptop")} {
raw, _ := json.Marshal(out)
if strings.Contains(string(raw), "secret-hash") || strings.Contains(string(raw), "$2a$") {
t.Fatalf("a hash in %s", raw)
}
}
us := ParseUsers([]byte(users))
if len(us) != 2 || len(us[0].Publish) != 2 || !us[0].AllowResponses || us[1].AllowResponses {
t.Fatalf("%+v", us)
}
}
func TestWhetherAUserMayDoSomething(t *testing.T) {
ok, err := UserCan([]byte(users), "laptop.node-tools", "publish", "$KV.claude-code_servers.all.x")
if err != nil || ok["allowed"] != true || ok["by"] != "$KV.claude-code_servers.>" {
t.Fatalf("%v %v", ok, err)
}
no, _ := UserCan([]byte(users), "laptop.node-tools", "publish", "$KV.claude-code_holdings.laptop")
if no["allowed"] != false {
t.Fatalf("%v", no)
}
if _, err := UserCan([]byte(users), "nobody", "publish", "x"); err == nil {
t.Fatal("an unknown user answered")
}
for _, c := range []struct {
p, s string
want bool
}{{"a.*.c", "a.b.c", true}, {"a.>", "a.b.c", true}, {"a.>", "a", false}, {"a.b", "a.b.c", false}, {"a.*", "a.b.c", false}} {
if SubjectMatches(c.p, c.s) != c.want {
t.Errorf("%s ~ %s", c.p, c.s)
}
}
}
// novox/hq issue 327: who opens and closes connections, per user and name, and which were dropped rather
// than closed by their client — read from the server's record of closed connections.
func TestClosedConnectionsArePerUserAndNameAndSayWhichWereDropped(t *testing.T) {
const closed = `{"connections":[
{"cid":10,"name":"mesh-controller verb conditions","authorized_user":"controller","start":"2026-10-08T14:12:00Z","stop":"2026-10-08T14:12:01Z","reason":"Client Closed"},
{"cid":11,"name":"mesh-controller verb conditions","authorized_user":"controller","start":"2026-10-08T14:13:00Z","stop":"2026-10-08T14:13:03Z","reason":"Client Closed"},
{"cid":12,"name":"ace.node-tools","authorized_user":"ace.node-tools","start":"2026-10-08T13:00:00Z","stop":"2026-10-08T14:14:00Z","reason":"Stale Connection"},
{"cid":13,"name":"ace.node-tools","authorized_user":"ace.node-tools","start":"2026-10-08T14:14:05Z","stop":"2026-10-08T14:15:00Z","reason":"Read Error"},
{"cid":5,"name":"old","authorized_user":"controller","start":"2026-10-08T10:00:00Z","stop":"2026-10-08T10:00:01Z","reason":"Client Closed"},
{"cid":14,"name":"","start":"2026-10-08T14:16:00Z","stop":"2026-10-08T14:16:00Z","reason":"Authentication Failure"}]}`
now, _ := time.Parse(time.RFC3339, "2026-10-08T14:20:00Z")
out, err := ClosedConnections([]byte(closed), now, 1, "")
if err != nil {
t.Fatal(err)
}
users := out["users"].([]ClosedUser)
if len(users) != 3 {
t.Fatalf("%+v", users)
}
byUser := map[string]ClosedUser{}
for _, u := range users {
byUser[u.User] = u
}
c := byUser["controller"]
if c.Closed != 2 || c.Dropped != 0 || c.ClosedPerHour != 2 || len(c.Names) != 1 ||
c.Names[0].Name != "mesh-controller verb conditions" || c.Names[0].Lived != 3 {
t.Errorf("the controller's: %+v", c)
}
tools := byUser["ace.node-tools"]
if tools.Closed != 2 || tools.Dropped != 2 || tools.Names[0].Reasons["Stale Connection"] != 1 {
t.Errorf("the runtime's: %+v", tools)
}
if byUser["(no user: refused before it logged in)"].Dropped != 1 {
t.Errorf("a refused login is not said: %+v", users)
}
if out["reaches"] != true {
t.Error("a record older than the window does not reach back the whole window")
}
narrowed, _ := ClosedConnections([]byte(closed), now, 1, "node-tools")
if got := narrowed["users"].([]ClosedUser); len(got) != 1 || got[0].User != "ace.node-tools" {
t.Errorf("narrowed: %+v", got)
}
}
// novox/hq issue 353: a grant written to the user list is in force only once the server reloaded after the
// write; anything not known is not in force.
func TestAGrantIsInForceOnlyWhenTheServerReloadedAfterTheListWasWritten(t *testing.T) {
written, err := ParseWritten("2026-10-09 17:55:56.422211788 +0200\n")
if err != nil {
t.Fatal(err)
}
varz := func(at string) []byte { return []byte(`{"config_load_time":"` + at + `"}`) }
allowed := func() map[string]any { return map[string]any{"allowed": true} }
cases := []struct {
name string
answer map[string]any
werr error
varz []byte
verr error
want bool
says string
}{
{"reloaded after the write", allowed(), nil, varz("2026-10-09T15:56:05.1Z"), nil, true, ""},
{"reloaded before the write", allowed(), nil, varz("2026-10-09T15:55:56.4Z"), nil, false, "has not reloaded"},
{"not allowed in the file", map[string]any{"allowed": false}, nil, varz("2026-10-09T15:56:05Z"), nil, false, "does not allow"},
{"the write time unknown", allowed(), errors.New("no container"), varz("2026-10-09T15:56:05Z"), nil, false, "no container"},
{"the server unknown", allowed(), nil, nil, errors.New("refused"), false, "could not be read"},
{"a server that says no load time", allowed(), nil, []byte(`{}`), nil, false, "does not say"},
}
for _, c := range cases {
got := InForce(c.answer, written, c.werr, c.varz, c.verr)
if got["in_force"] != c.want || (c.says != "" && !strings.Contains(fmt.Sprint(got["why"]), c.says)) {
t.Errorf("%s: %v", c.name, got)
}
}
if _, err := ParseWritten("yesterday"); err == nil {
t.Error("a write time that is not one was read")
}
}
// **The tools that change the bus are two, each refuses before it reaches the bus, and every other tool reads**
// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them, but to read an accepted module's durable's
// twin (H-A-472), and a twin that cannot be read refuses.
func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) {
bus := func(context.Context) (jetstream.JetStream, func(), error) {
return nil, nil, errors.New("the bus is opened here only to read a twin, and cannot be")
}
said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil }
newForm := func() (map[string]keeper.Asked, error) {
made := map[string]keeper.Made{}
for _, e := range []catalogue.BusStateEntry{{Name: "log", Storage: "stream", Template: "log"}, {Name: "drafts", Storage: "key-value", Template: "state"}} {
p, err := keeper.PlanOf("m", e)
if err != nil {
t.Fatal(err)
}
made[e.Name] = keeper.Made{Plan: p, Created: true}
}
return map[string]keeper.Asked{"m": {Module: "m", Made: made}}, nil
}
// The keeper's accepted set (its state plans, §538): m, which asks for kept data, and quiet, which asks for none and
// so has no key in asked (review round 2: a module keeping no bus data still has live durables).
accepted := func() (map[string]keeper.Planned, error) {
return map[string]keeper.Planned{"m": {Module: "m"}, "quiet": {Module: "quiet"}}, nil
}
for _, c := range []struct {
tool string
args map[string]any
want string
}{
{"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller"}, "why is required"},
{"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": " "}, "why is required"},
{"nats_consumer_remove", map[string]any{"stream": "A.B", "consumer": "c", "why": "x"}, "stream and consumer are required"},
{"nats_consumer_remove", map[string]any{"stream": "S", "consumer": ">", "why": "x"}, "stream and consumer are required"},
{"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14"}, "why is required"},
{"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "why": "x"}, "exactly one of node"},
{"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14", "prefix": "g", "why": "x"}, "exactly one of node"},
{"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "prefix": "g*", "why": "x"}, "without wildcards"},
{"nats_bucket_remove", map[string]any{"bucket": "", "node": "g14", "why": "x"}, "bucket is required"},
{"nats_stream_remove", map[string]any{"stream": "ASSIGNMENTS"}, "why is required"},
{"nats_stream_remove", map[string]any{"stream": "ASSIGNMENTS", "why": " "}, "why is required"},
{"nats_stream_remove", map[string]any{"stream": "A.>", "why": "x"}, "stream is required"},
{"nats_stream_remove", map[string]any{"stream": "", "why": "x"}, "stream is required"},
// CAT-N9: an entry of the new form goes only by drop or recreate; an old section's goes as before.
{"nats_stream_remove", map[string]any{"stream": "m_log", "why": "x"}, "removed only by nats_drop"},
{"nats_bucket_remove", map[string]any{"bucket": "m_drafts", "node": "g14", "why": "x"}, "m's kept data \"drafts\""},
// Design 47 H, H-A-127: a consumer is judged by its name: an accepted module's durable is refused on any stream.
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_m", "why": "x"}, "a module of an accepted manifest"},
{"nats_consumer_remove", map[string]any{"stream": "m_log", "consumer": "ace_m", "why": "x"}, "a module of an accepted manifest"},
// H-A-128: a durable beginning <node>_<module>_ is the module's too.
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_m_keeper", "why": "x"}, "a module of an accepted manifest"},
// H-A-128: an accepted module that asks for no kept data is refused too, its durables and its suffixed ones.
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet", "why": "x"}, "a module of an accepted manifest"},
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "a module of an accepted manifest"},
// H-A-472 (1): a twin that cannot be read keeps the durable refused, saying so.
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_m", "why": "x"}, "its twin novox_m_tools on EVENTS cannot be read"},
{"nats_consumer_remove", map[string]any{"stream": "EVENTS", "consumer": "novox_quiet_tools", "why": "x"}, "its twin novox_quiet on EVENTS cannot be read"},
} {
var run func(map[string]any) (any, error)
for _, tl := range removalTools(bus, said, newForm, accepted, nil, nil) {
if tl.Name == c.tool {
run = tl.Run
}
}
if run == nil {
t.Fatalf("no tool %s", c.tool)
}
if _, err := run(c.args); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s %v: %v, want a refusal saying %q", c.tool, c.args, err, c.want)
}
}
// H-A-472 (1): a module's durable whose twin (<node>_<module> and <node>_<module>_tools, A-340) is current goes; the
// twin is current with pulls waiting and a delivery within the day; a durable with another suffix has no twin.
now := time.Date(2026, 11, 22, 14, 0, 0, 0, time.UTC)
hourAgo, twoDaysAgo := now.Add(-time.Hour), now.Add(-48*time.Hour)
day := DefaultTwinCurrentHours * time.Hour
for _, c := range []struct {
name string
info *jetstream.ConsumerInfo
within time.Duration
want bool
}{
{"pulls waiting, delivered an hour ago", &jetstream.ConsumerInfo{NumWaiting: 1, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, true},
{"no pull waiting", &jetstream.ConsumerInfo{Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, day, false},
{"delivered two days ago", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, day, false},
{"delivered two days ago, the setting at 72 hours", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &twoDaysAgo}}, 72 * time.Hour, true},
{"delivered an hour ago, the setting at 1 hour", &jetstream.ConsumerInfo{NumWaiting: 2, Delivered: jetstream.SequenceInfo{Last: &hourAgo}}, time.Hour, false},
{"never delivered", &jetstream.ConsumerInfo{NumWaiting: 2}, day, false},
{"not read", nil, day, false},
} {
if got := current(c.info, now, c.within); got != c.want {
t.Errorf("current, %s: %v, want %v", c.name, got, c.want)
}
}
// H-A-472 third addendum (2): the setting twin-current-hours, in hours; its default 24 when the configuration gives
// none or no positive whole number.
for _, c := range []struct {
raw string
err error
want time.Duration
}{
{"72", nil, 72 * time.Hour}, {"1", nil, time.Hour}, {"", errors.New("no configuration"), 24 * time.Hour},
{"0", nil, 24 * time.Hour}, {"-3", nil, 24 * time.Hour}, {`"x"`, nil, 24 * time.Hour},
} {
if got := twinCurrentWithin(json.RawMessage(c.raw), c.err); got != c.want {
t.Errorf("twin-current-hours %q (%v): %s, want %s", c.raw, c.err, got, c.want)
}
}
for _, c := range []struct{ module, durable, want string }{
{"m", "m", "novox_m_tools"}, {"m", "m_tools", "novox_m"}, {"m", "m_keeper", ""}, {"m", "m_tools_x", ""},
} {
if got := twinOf("novox", c.module, c.durable); got != c.want {
t.Errorf("the twin of novox_%s: %q, want %q", c.durable, got, c.want)
}
}
// An orphan on a module's kept-data stream is not refused by its stream (H-A-126): the call reaches the bus.
unreachable := func(context.Context) (jetstream.JetStream, func(), error) {
return nil, nil, errors.New("the bus is not reached here")
}
for _, tl := range removalTools(unreachable, said, newForm, accepted, nil, nil) {
if tl.Name != "nats_consumer_remove" {
continue
}
for _, orphan := range []string{"c", "novox_mx", "novox_mx_keeper"} {
if _, err := tl.Run(map[string]any{"stream": "m_log", "consumer": orphan, "why": "an orphan"}); err == nil || !strings.Contains(err.Error(), "not reached") {
t.Errorf("the orphan %s on m_log: %v, want the call to reach the bus", orphan, err)
}
}
}
// The read tools stay read tools: none of the first ten takes a `why`, and the list is these ten.
for _, tl := range tools(nil, nil, nil) {
if _, ok := tl.Input["why"]; ok || strings.Contains(tl.Name, "remove") {
t.Errorf("%s changes the bus; only the removal tools may", tl.Name)
}
}
}
// The keeper's health reads what waits on its own consumer from the shared monitoring read: that read asks for the
// consumers, or the health answers "cannot be read" for ever (2026-11-08, issue 1966).
func TestTheHeldReadAsksForTheConsumers(t *testing.T) {
if !strings.Contains(heldJsz, "consumers=1") {
t.Fatalf("the monitoring read %q asks for no consumers, so keeperPending never finds the keeper's", heldJsz)
}
doc := []byte(`{"account_details":[{"stream_detail":[{"name":"EVENTS","consumer_detail":[{"name":"novox_nats_keeper","num_pending":2}]}]}]}`)
if got := keeperPending(doc); got != 2 {
t.Fatalf("keeperPending = %d on a read with the keeper's consumer, want 2", got)
}
}