Serve the read verbs on the serving controller's own connection, and name every connection

Each verb ran as a process that dialled the bus, so hundreds of short
connections an hour, all named mesh-controller, hid any client reconnecting
in a loop (hq issue 327). D15 now says a user whose connections keep dropping.

cmd/mesh-controller/reconnects_test.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// The serving controller's own connection serves what it does for a moment: no connection is opened,
+// and closing what it was lent leaves the serving one open (novox/hq issue 327).
+func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
+	bus := testbus.Start(t)
+	serving, err := broker.Dial(bus.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer serving.Close()
+	if err := serving.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	was := servingBus.Load()
+	servingBus.Store(serving)
+	defer servingBus.Store(was)
+	t.Setenv(broker.NATSVar, bus.ClientURL())
+
+	before, _ := bus.Varz(nil)
+	lent, err := aBus()
+	if err != nil {
+		t.Fatal(err)
+	}
+	lent.Close()
+	if !serving.Conn().IsConnected() {
+		t.Fatal("closing a lent connection closed the serving controller's")
+	}
+	if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil {
+		t.Fatal(err)
+	}
+	handlers, _, err := seatToolHandlers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
+		t.Fatalf("hand-acts answered %+v", a)
+	}
+	_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
+	after, _ := bus.Varz(nil)
+	if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
+		t.Fatalf("the serving controller opened %d connection(s) of its own", opened)
+	}
+}
+
+// A verb that still runs as a process of its own says which in the bus's list of connections.
+func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
+	bus := testbus.Start(t)
+	setup, err := broker.Dial(bus.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := setup.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	setup.Close()
+	asAProcess(t, bus.ClientURL())
+	handlers, _, err := seatToolHandlers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A silence acts, so it is still its own process; it dials, and finds no such condition.
+	_, _ = handlers["conditions"](context.Background(),
+		json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
+	names := closedNames(t, bus)
+	found := false
+	for _, n := range names {
+		found = found || n == "mesh-controller verb conditions"
+	}
+	if !found {
+		t.Fatalf("the verb's connection was named %q", names)
+	}
+}
+
+func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) {
+	for _, c := range []struct{ command, verb, want string }{
+		{"serve", "", "mesh-controller serving"},
+		{"conditions", "conditions", "mesh-controller verb conditions"},
+		{"delivery", "delivery-check", "mesh-controller verb delivery-check"},
+		{"push", "", "mesh-controller command push"},
+	} {
+		if got := connectionName(c.command, c.verb); got != c.want {
+			t.Errorf("%s/%s: %q", c.command, c.verb, got)
+		}
+	}
+}
+
+// D15: a user whose connections are dropped more than twelve times an hour is said, in plain words; one
+// whose client closes its own short connections is not.
+func TestAClientReconnectingInALoopIsSaid(t *testing.T) {
+	var read closedConnections
+	if err := json.Unmarshal([]byte(`{"hours":1,"reaches":true,"users":[
+		{"user":"ace.node-tools","closed":40,"dropped":40,"dropped_per_hour":40,"names":[
+			{"name":"ace.node-tools","closed":40,"dropped":40,"reasons":{"Stale Connection":30,"Read Error":10}}]},
+		{"user":"controller","closed":300,"dropped":0,"names":[
+			{"name":"mesh-controller verb delivery-check","closed":300,"dropped":0,"reasons":{"Client Closed":300}}]},
+		{"user":"node.anchor","closed":5,"dropped":5,"names":[{"name":"mesh-host/anchor","closed":5,"dropped":5,
+			"reasons":{"Read Error":5}}]}]}`), &read); err != nil {
+		t.Fatal(err)
+	}
+	found := reconnecting(read)
+	if len(found) != 1 {
+		t.Fatalf("%+v", found)
+	}
+	o := found[0]
+	if o.ID != "ace.node-tools" || o.Machine != "ace" || o.Kind != kindBusReconnects ||
+		o.Headline != "The tool runner on ace keeps losing the bus" || !strings.Contains(o.Said, "Stale Connection ×30") {
+		t.Fatalf("%+v", o)
+	}
+	if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
+		Resolved: o.Resolved}, "ace"); !ok {
+		t.Fatalf("not plain: %s", why)
+	}
+}
+
+// The bus's module is asked on its machine, over the bus.
+func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) {
+	bus := testbus.Start(t)
+	conn, err := nats.Connect(bus.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) {
+		_ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	read, err := askClosed(context.Background(), conn, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(read.Users) != 1 || read.Users[0].Closed != 2 {
+		t.Fatalf("%+v", read)
+	}
+}