mesh-catalog/modules/nats/cmd/nats-tools/watch_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"
)

// The watch (novox/hq ADR 0357, issue 270): every user's slow consumer and every connection the bus closed
// for breaking its rules, said once as an event the controller raises as S9.

var watchNow = time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)

// closedRecord is a record of closed connections as the server serves it, with a password-shaped field the
// server never serves, to prove nothing but the user is carried.
const closedRecord = `{"connections":[
 {"cid":10,"name":"g14.claude-code","authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:58:00Z","reason":"Slow Consumer (Write Deadline)","pending_bytes":334000},
 {"cid":11,"name":"mesh-controller verb node","authorized_user":"controller","start":"2026-10-14T11:58:00Z","stop":"2026-10-14T11:58:01Z","reason":"Client Closed"},
 {"cid":12,"name":"mesh-host/g14","authorized_user":"node.g14","start":"2026-10-14T10:00:00Z","stop":"2026-10-14T11:59:00Z","reason":"Stale Connection"},
 {"cid":13,"name":"g14.node-tools","authorized_user":"g14.node-tools","start":"2026-10-14T10:00:00Z","stop":"2026-10-14T11:59:10Z","reason":"Read Error"},
 {"cid":14,"name":"shanks.desk","authorized_user":"shanks.desk","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:20Z","reason":"Maximum Message Payload Exceeded"},
 {"cid":15,"name":"","authorized_user":"","start":"2026-10-14T11:59:00Z","stop":"2026-10-14T11:59:30Z","reason":"Authentication Failure"},
 {"cid":16,"name":"ace.media","authorized_user":"ace.media","start":"2026-10-14T09:00:00Z","stop":"2026-10-14T10:30:00Z","reason":"Slow Consumer (Pending Bytes)"},
 {"cid":17,"name":"ace.future","authorized_user":"ace.future","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:40Z","reason":"A Reason Not Yet Invented","password":"never-carried"},
 {"cid":18,"name":"ace.route-proxy","authorized_user":"ace.route-proxy","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:50Z","reason":"Server Shutdown"}
]}`

type published struct {
	event string
	body  Closure
}

type recorder struct {
	mu   sync.Mutex
	got  []published
	fail error
}

func (r *recorder) emit(event string, body any) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.fail != nil {
		return r.fail
	}
	raw, _ := json.Marshal(body)
	if strings.Contains(string(raw), "never-carried") || strings.Contains(strings.ToLower(string(raw)), "password") {
		panic("a credential was published: " + string(raw))
	}
	var c Closure
	_ = json.Unmarshal(raw, &c)
	r.got = append(r.got, published{event, c})
	return nil
}

func (r *recorder) said() []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	var out []string
	for _, p := range r.got {
		out = append(out, p.event+" "+p.body.User)
	}
	sort.Strings(out)
	return out
}

func recordOf(doc *string) Monitor {
	return func(_ context.Context, path string) ([]byte, error) {
		if !strings.HasPrefix(path, "/connz?state=closed&auth=1") {
			return nil, errors.New("the watch asked " + path)
		}
		return []byte(*doc), nil
	}
}

func watchAt(doc *string, r *recorder) *Watch {
	w := NewWatch(recordOf(doc), r.emit)
	w.now = func() time.Time { return watchNow }
	return w
}

func TestTheWatchSaysEachClosureTheServerDroppedNamingItsUserAndNothingRoutine(t *testing.T) {
	doc := closedRecord
	r := &recorder{}
	watchAt(&doc, r).Look(context.Background())
	want := []string{
		"bus.connection-closed ace.future",  // a reason the module does not know is said
		"bus.connection-closed shanks.desk", // a limit broken
		"bus.slow-consumer g14.claude-code", // the slow consumer research 036 could not name
	}
	if got := r.said(); strings.Join(got, "|") != strings.Join(want, "|") {
		t.Fatalf("said %v, want %v (none for the client's own close, the network, a refused login, the "+
			"server's shutdown, or a closure older than the hour)", got, want)
	}
	for _, p := range r.got {
		if p.body.User == "g14.claude-code" {
			if p.body.CID != 10 || p.body.Reason != "Slow Consumer (Write Deadline)" || p.body.Pending != 334000 ||
				!p.body.Closed.Equal(time.Date(2026, 10, 14, 11, 58, 0, 0, time.UTC)) || p.body.Lived != 18*60 {
				t.Fatalf("the slow consumer said as %+v", p.body)
			}
		}
	}
}

func TestTheWatchSaysAClosureOnceWhileItRuns(t *testing.T) {
	doc := closedRecord
	r := &recorder{}
	w := watchAt(&doc, r)
	w.Look(context.Background())
	w.Look(context.Background())
	if got := len(r.said()); got != 3 {
		t.Fatalf("said %d events over two looks at the same record, want 3: %v", got, r.said())
	}
}

func TestAClosureWhosePublishFailedIsTriedAgainAtTheNextLook(t *testing.T) {
	doc := closedRecord
	r := &recorder{fail: errors.New("the runtime did not answer")}
	w := watchAt(&doc, r)
	w.Look(context.Background())
	if len(r.said()) != 0 {
		t.Fatal("a failed publish was counted as said")
	}
	if report := w.Report(); len(report["not_published"].(map[string]string)) != 3 {
		t.Fatalf("the watch does not say what it could not publish: %v", report)
	}
	r.fail = nil
	w.Look(context.Background())
	if got := len(r.said()); got != 3 {
		t.Fatalf("said %d after the runtime came back, want the 3 that failed", got)
	}
	if _, left := w.Report()["not_published"]; left {
		t.Fatal("a closure published at last is still said not published")
	}
}

func TestAWatchStartedAgainSaysTheLastHourAgain(t *testing.T) {
	doc := closedRecord
	first := &recorder{}
	watchAt(&doc, first).Look(context.Background())
	again := &recorder{}
	watchAt(&doc, again).Look(context.Background())
	if strings.Join(first.said(), "|") != strings.Join(again.said(), "|") || len(again.said()) != 3 {
		t.Fatalf("a restarted watch said %v, the first %v: the last hour must be said again, for the controller "+
			"to count once", again.said(), first.said())
	}
}

func TestAReusedConnectionIDWithANewOpeningIsANewClosure(t *testing.T) {
	doc := `{"connections":[{"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:50:00Z","reason":"Slow Consumer (Pending Bytes)"}]}`
	r := &recorder{}
	w := watchAt(&doc, r)
	w.Look(context.Background())
	// The server restarted and numbered connections from one again.
	doc = `{"connections":[
 {"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:50:00Z","reason":"Slow Consumer (Pending Bytes)"},
 {"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T11:55:00Z","stop":"2026-10-14T11:59:00Z","reason":"Slow Consumer (Pending Bytes)"}]}`
	w.Look(context.Background())
	if got := len(r.said()); got != 2 {
		t.Fatalf("said %d, want 2: the same connection id opened again is another closure", got)
	}
}

func TestTheWatchSaysItCannotReadTheRecord(t *testing.T) {
	r := &recorder{}
	w := NewWatch(func(context.Context, string) ([]byte, error) { return nil, errors.New("connection refused") }, r.emit)
	w.Look(context.Background())
	if report := w.Report(); !strings.Contains(report["last_error"].(string), "connection refused") {
		t.Fatalf("a record that could not be read is not said: %v", report)
	}
}

func TestTheManifestDeclaresTheToolsServedTheWatchesEventsAndItsInterval(t *testing.T) {
	raw, err := os.ReadFile("../../module.json")
	if err != nil {
		t.Fatal(err)
	}
	var m struct {
		Tools []string `json:"tools"`
		Emits []string `json:"emits"`
	}
	if err := json.Unmarshal(raw, &m); err != nil {
		t.Fatal(err)
	}
	var served []string
	for _, tool := range tools(nil, nil, nil, NewWatch(nil, nil)) {
		served = append(served, tool.Name)
	}
	sort.Strings(served)
	sort.Strings(m.Tools)
	if strings.Join(served, ",") != strings.Join(m.Tools, ",") {
		t.Fatalf("served %v, the manifest lists %v", served, m.Tools)
	}
	sort.Strings(m.Emits)
	if strings.Join(m.Emits, ",") != EventConnectionClosed+","+EventSlowConsumer {
		t.Fatalf("the manifest emits %v: the runtime may publish only what is declared", m.Emits)
	}
	if WatchEvery != 30*time.Second || WatchSaysAgain != time.Hour {
		t.Fatalf("the watch looks every %s and reads back %s; ADR 0357 says 30s and an hour", WatchEvery, WatchSaysAgain)
	}
}