mesh-controller/cmd/mesh-controller/reconnects_test.go

package main

import (
	"context"
	"encoding/json"
	"strings"
	"testing"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
	"github.com/novox/mesh-controller/internal/testbus"
)

// The serving controller's own connection serves what it does for a moment: no connection is opened,
// and closing what it was lent leaves the serving one open (novox/hq issue 327).
func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
	bus := testbus.Start(t)
	serving, err := broker.Dial(bus.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	defer serving.Close()
	if err := serving.EnsureControllerBuckets(); err != nil {
		t.Fatal(err)
	}
	was := servingBus.Load()
	servingBus.Store(serving)
	defer servingBus.Store(was)
	t.Setenv(broker.NATSVar, bus.ClientURL())

	before, _ := bus.Varz(nil)
	lent, err := aBus()
	if err != nil {
		t.Fatal(err)
	}
	lent.Close()
	if !serving.Conn().IsConnected() {
		t.Fatal("closing a lent connection closed the serving controller's")
	}
	if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil {
		t.Fatal(err)
	}
	handlers, _, err := seatToolHandlers()
	if err != nil {
		t.Fatal(err)
	}
	answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`))
	if err != nil {
		t.Fatal(err)
	}
	if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
		t.Fatalf("hand-acts answered %+v", a)
	}
	_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
	after, _ := bus.Varz(nil)
	if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
		t.Fatalf("the serving controller opened %d connection(s) of its own", opened)
	}
}

// A verb that still runs as a process of its own says which in the bus's list of connections.
func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
	bus := testbus.Start(t)
	setup, err := broker.Dial(bus.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	if err := setup.EnsureControllerBuckets(); err != nil {
		t.Fatal(err)
	}
	setup.Close()
	asAProcess(t, bus.ClientURL())
	handlers, _, err := seatToolHandlers()
	if err != nil {
		t.Fatal(err)
	}
	// A silence acts, so it is still its own process; it dials, and finds no such condition.
	_, _ = handlers["conditions"](context.Background(),
		json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
	names := closedNames(t, bus)
	found := false
	for _, n := range names {
		found = found || n == "mesh-controller verb conditions"
	}
	if !found {
		t.Fatalf("the verb's connection was named %q", names)
	}
}

func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) {
	for _, c := range []struct{ command, verb, want string }{
		{"serve", "", "mesh-controller serving"},
		{"conditions", "conditions", "mesh-controller verb conditions"},
		{"delivery", "delivery-check", "mesh-controller verb delivery-check"},
		{"push", "", "mesh-controller command push"},
	} {
		if got := connectionName(c.command, c.verb); got != c.want {
			t.Errorf("%s/%s: %q", c.command, c.verb, got)
		}
	}
}

// D15: a user whose connections are dropped more than twelve times an hour is said, in plain words; one
// whose client closes its own short connections is not.
func TestAClientReconnectingInALoopIsSaid(t *testing.T) {
	var read closedConnections
	if err := json.Unmarshal([]byte(`{"hours":1,"reaches":true,"users":[
		{"user":"ace.node-tools","closed":40,"dropped":40,"dropped_per_hour":40,"names":[
			{"name":"ace.node-tools","closed":40,"dropped":40,"reasons":{"Stale Connection":30,"Read Error":10}}]},
		{"user":"controller","closed":300,"dropped":0,"names":[
			{"name":"mesh-controller verb delivery-check","closed":300,"dropped":0,"reasons":{"Client Closed":300}}]},
		{"user":"(no user: refused before it logged in)","closed":90,"dropped":90,"names":[{"name":"","closed":90,
			"dropped":90,"reasons":{"Authentication Failure":90}}]},
		{"user":"node.anchor","closed":5,"dropped":5,"names":[{"name":"mesh-host/anchor","closed":5,"dropped":5,
			"reasons":{"Read Error":5}}]}]}`), &read); err != nil {
		t.Fatal(err)
	}
	found := reconnecting(read)
	if len(found) != 1 {
		t.Fatalf("%+v", found)
	}
	o := found[0]
	if o.ID != "ace.node-tools" || o.Machine != "ace" || o.Kind != kindBusReconnects ||
		o.Headline != "The tool runner on ace keeps losing the bus" || !strings.Contains(o.Said, "Stale Connection ×30") {
		t.Fatalf("%+v", o)
	}
	if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
		Resolved: o.Resolved}, "ace"); !ok {
		t.Fatalf("not plain: %s", why)
	}
}

// The bus's module is asked on its machine, over the bus.
func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) {
	bus := testbus.Start(t)
	conn, err := nats.Connect(bus.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) {
		_ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`))
	})
	if err != nil {
		t.Fatal(err)
	}
	defer func() { _ = sub.Unsubscribe() }()
	read, err := askClosed(context.Background(), conn, "anchor")
	if err != nil {
		t.Fatal(err)
	}
	if len(read.Users) != 1 || read.Users[0].Closed != 2 {
		t.Fatalf("%+v", read)
	}
}

// A bus module older than the tool answers nothing to the question: the probe passes over it quietly.
func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) {
	bus := testbus.Start(t)
	conn, err := nats.Connect(bus.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	found, err := reconnectsOn(context.Background(), conn, "anchor")
	if err != nil || len(found) != 0 {
		t.Fatalf("a bus module without the tool: %v %v", found, err)
	}
}

// A verb answered in the serving controller says its flag errors in its answer, not in the controller's log.
func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) {
	bus := testbus.Start(t)
	serving, err := broker.Dial(bus.ClientURL())
	if err != nil {
		t.Fatal(err)
	}
	defer serving.Close()
	was := servingBus.Load()
	servingBus.Store(serving)
	defer servingBus.Store(was)
	answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"})
	if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") {
		t.Fatalf("answered %+v", answer)
	}
}