mesh-host/internal/bootstrap/enrol_test.go

package bootstrap

import (
	"context"
	"crypto/ed25519"
	"fmt"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/identity"
	"github.com/novox/mesh-host/internal/system"
)

// Step 6 is where the mesh stops being something running on a machine and starts being something
// the machine belongs to. What these tests defend is that it cannot happen twice, and that
// "installed" is never claimed for a machine the mesh has not actually heard from.

// alreadyEnrolled writes an identity file, as `mesh-host enrol` leaves behind.
func alreadyEnrolled(t *testing.T, node string) string {
	t.Helper()
	state := filepath.Join(t.TempDir(), "state.json")
	mine, err := identity.Generate(node)
	if err != nil {
		t.Fatal(err)
	}
	// A whole membership, because an identity that cannot reach its mesh is refused on the way in
	// — which is the right refusal and not the one being tested here.
	signer, _, err := ed25519.GenerateKey(nil)
	if err != nil {
		t.Fatal(err)
	}
	mine.Membership = identity.Membership{
		Broker: "192.0.2.10:5671", Fingerprint: "sha256:whatever",
		Signer: signer, Password: "issued-at-enrolment",
	}
	if err := identity.Save(identity.Path(state), mine); err != nil {
		t.Fatal(err)
	}
	return state
}

func arch(t *testing.T) system.System {
	t.Helper()
	chosen, err := system.For("arch")
	if err != nil {
		t.Fatal(err)
	}
	return chosen
}

// A machine that has already enrolled is not enrolled again, and no token is spent on it. The
// installer is run over and over; a second identity is one the mesh does not know, and the mesh
// believes the first.
func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
	runtime := &asked{answer: func(name string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "node list"):
			return "anchor               here           01J0\n", nil
		// The bus is the nats module already, so its users are the module's.
		case strings.Contains(joined, "module list"):
			return "nats                 1\npostgres             1\n", nil
		// An agent is running here too. Both halves are needed: enrolling makes the mesh hear from
		// a machine once, so the first answer alone also describes a machine with no agent at all.
		case name == "pgrep":
			return "4242\n", nil
		}
		return "", fmt.Errorf("unexpected: %s %v", name, args)
	}}

	out, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	if out.Joined || out.Added {
		t.Error("a machine that had already enrolled enrolled again")
	}
	if runtime.ran("token issue") {
		t.Errorf("a token was issued for a machine that already holds an identity: %v",
			runtime.commands)
	}
	// Nor are the bus's users placed: the bus is adopted, so they are the nats module's, and the
	// installer's bridge is not run again (novox/hq ADR 0331 §5).
	if runtime.ran("broker accounts") || runtime.ran("accounts.conf") {
		t.Errorf("the bus's users were placed on a re-run: %v", runtime.commands)
	}
	if out.Agent != "already running" {
		t.Errorf("the host agent is reported as %q", out.Agent)
	}
}

// The mesh having heard from a machine is not the same as an agent running on it, and enrolling is
// itself the thing that makes the mesh hear. Taken as proof of life it ends the step believing an
// agent it never started, and everything after is the control plane being told things that never
// reach the machine: the next push is accepted, recorded, and applied by nobody.
func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) {
	runtime := &asked{answer: func(name string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "module list"):
			return "nats                 1\n", nil // the bus is adopted: no users to place
		case strings.Contains(joined, "node list"):
			// Exactly what enrolling leaves behind, with nothing running.
			return "anchor               here           01J0\n", nil
		case name == "pgrep":
			return "", fmt.Errorf("exit status 1")
		case name == "sh":
			return "", nil
		}
		return "", fmt.Errorf("unexpected: %s %v", name, args)
	}}

	out, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	if out.Agent == "already running" {
		t.Fatal("a machine with no agent was reported as already running it")
	}
	if !runtime.ran("nohup") {
		t.Errorf("no agent was started on a machine that has none: %v", runtime.commands)
	}
}

// A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling
// replaces the identity the mesh recorded, which is a deliberate act and not something an
// installer does on its own.
func TestAMachineEnrolledUnderAnotherNameIsRefused(t *testing.T) {
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if strings.Contains(strings.Join(args, " "), "node list") {
			return "somewhere-else       here           01J0\n", nil
		}
		return "", nil
	}}

	_, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "somewhere-else"), Timeout: time.Second,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err == nil {
		t.Fatal("a machine already enrolled as something else was enrolled again")
	}
	for _, wanted := range []string{"somewhere-else", "--node"} {
		if !strings.Contains(err.Error(), wanted) {
			t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
		}
	}
}

// **The mesh having heard from the node is the proof, not a process existing.** A host that is
// running and cannot reach the broker looks exactly like a successful install until the first push
// silently applies nothing — which is the class of fault this whole program exists to stop being
// found late.
func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "module list"):
			return "nats                 1\n", nil // the bus is adopted: no users to place
		case strings.Contains(joined, "node list"):
			// Enrolled, and never spoken.
			return "anchor               never spoken    01J0\n", nil
		case args[0] == "show":
			return "LoadState=loaded\nActiveState=active\n", nil
		case args[0] == "is-enabled":
			return "enabled\n", nil
		}
		return "", nil
	}}

	_, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service",
		Timeout: time.Second, Wait: 0,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err == nil {
		t.Fatal("a node the mesh has never heard from was reported enrolled and running")
	}
	if !strings.Contains(err.Error(), "MESH_BROKER_ADDRESS") {
		t.Errorf("the failure does not name the thing that is silently fatal when wrong:\n%v", err)
	}
}

// A machine with no service to start is refused, and the refusal says what is missing rather than
// inventing a unit file. What a unit says is a packaging decision, and an installer writing one
// would put a file on the machine that whatever installed the host will disagree with.
func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) {
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "module list"):
			return "nats                 1\n", nil // the bus is adopted: no users to place
		case strings.Contains(joined, "node list"):
			return "anchor               never spoken    01J0\n", nil
		case args[0] == "show":
			// systemd knows nothing about it.
			return "LoadState=not-found\nActiveState=inactive\n", nil
		}
		return "", nil
	}}

	_, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service",
		Timeout: time.Second, Wait: 0,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err == nil {
		t.Fatal("a machine with no host service was reported as having a running host")
	}
	for _, wanted := range []string{"mesh-host.service", "--host-in-background"} {
		if !strings.Contains(err.Error(), wanted) {
			t.Errorf("the refusal does not mention %q:\n%v", wanted, err)
		}
	}
}

// A machine with no name is refused before anything is said to the mesh. The name is what the
// record, the token, the assignment and the push all name, and one the installer invented would
// match nothing anybody types anywhere else.
func TestAMachineWithNoNameIsRefusedBeforeAnythingIsAsked(t *testing.T) {
	runtime := &asked{answer: func(string, []string) (string, error) {
		return "", fmt.Errorf("nothing should have been asked")
	}}
	_, err := Enrol(context.Background(), Options{Timeout: time.Second}, arch(t),
		controlPlane{container: "temp-mesh-controller", run: runtime.run}, func(string) {})
	if err == nil {
		t.Fatal("a machine with no name was enrolled")
	}
	if len(runtime.commands) != 0 {
		t.Errorf("the mesh was asked something first: %v", runtime.commands)
	}
}

// The token is found in what the mesh said, by the rule the lab uses: the one long unbroken line.
// The installer does not parse a format the control plane owns.
func TestTheTokenIsFoundInWhatTheMeshSaid(t *testing.T) {
	said := "a token for anchor, good once:\n\n  " + strings.Repeat("t", 240) + "\n\n" +
		"carry it to the machine and run: mesh-host enrol --token <token>\n"
	token, err := tokenIn(said)
	if err != nil {
		t.Fatal(err)
	}
	if token != strings.Repeat("t", 240) {
		t.Errorf("the token was read as %q", token)
	}

	if _, err := tokenIn("nothing here that looks like one\n"); err == nil {
		t.Fatal("an answer with no token in it was accepted")
	}
}

// A listing's name is matched as a whole word at the start of a line, so `registry` is not found
// inside `registry-mirror`. A substring match would report a module installed that is not, and the
// installer would skip creating it.
func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) {
	listing := "registry-mirror     1        built abc\nother               1        built def\n"
	if mentions(listing, "registry") {
		t.Error("registry-mirror was read as registry")
	}
	if !mentions(listing, "registry-mirror") {
		t.Error("registry-mirror was not found")
	}
}

// **Genesis places the bus's users, before the machine enrols and again after** (novox/hq
// 04-ISSUES/146). The enrolment account exists only in the mesh's records until somebody writes it
// beside the bus; so does the node's own, minted as it enrols. Without the first, the machine is
// refused by the bus it just raised; without the second, its agent is.
func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
	enrolled := false
	runtime := &asked{answer: func(name string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "node list"):
			if enrolled {
				return "anchor               here           01J0\n", nil
			}
			return "", nil
		case strings.Contains(joined, "node add"):
			return "added anchor\n", nil
		case strings.Contains(joined, "token issue"):
			return "a token for anchor, good once:\n\n  " + strings.Repeat("t", 240) + "\n\n", nil
		case strings.Contains(joined, "broker accounts"):
			return "accounts {\n  MESH { users = [] }\n}\n", nil
		case name == "/usr/local/bin/mesh-host":
			enrolled = true
			return "enrolled as anchor\n", nil
		case name == "pgrep", name == "sh":
			return "", nil
		}
		return "", fmt.Errorf("unexpected: %s %v", name, args)
	}}

	if _, err := Enrol(context.Background(), Options{
		Node: "anchor", State: filepath.Join(t.TempDir(), "state.json"), Timeout: time.Second,
		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {}); err != nil {
		t.Fatal(err)
	}

	placed, enrol := []int{}, -1
	for i, c := range runtime.commands {
		if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") &&
			strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") {
			placed = append(placed, i)
		}
		if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
			enrol = i
		}
	}
	if enrol < 0 || len(placed) != 2 || placed[0] > enrol || placed[1] < enrol {
		t.Fatalf("the bus's users were not placed before the machine enrolled and again after "+
			"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
	}
}

// A re-run on a machine that enrolled and whose bus is not yet the nats module places the
// controller's list again (novox/hq ADR 0331 §5, review of mesh-host #96). The apply just wrote the
// bundle's list — the controller alone — over the one placed at enrolment; left so, the next time the
// bus re-reads it every node and the enrolment account are refused.
func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *testing.T) {
	runtime := &asked{answer: func(name string, args []string) (string, error) {
		joined := strings.Join(args, " ")
		switch {
		case strings.Contains(joined, "node list"):
			return "anchor               here           01J0\n", nil
		case strings.Contains(joined, "module list"):
			return "mesh-controller      1\npostgres             1\n", nil
		case strings.Contains(joined, "broker accounts"):
			return "accounts {\n  MESH { users = [ { user: \"node.anchor\" } ] }\n}\n", nil
		case name == "pgrep":
			return "4242\n", nil
		case name == "sh":
			return "", nil
		}
		return "", fmt.Errorf("unexpected: %s %v", name, args)
	}}

	out, err := Enrol(context.Background(), Options{
		Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
		Host: "/usr/local/bin/mesh-host", HostInBackground: true,
	}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
		func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	if out.Joined || runtime.ran("token issue") {
		t.Errorf("a re-run enrolled again: %v", runtime.commands)
	}
	placed := false
	for _, c := range runtime.commands {
		if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") &&
			strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") {
			placed = true
		}
	}
	if !placed {
		t.Fatalf("the controller's list was not placed again on a re-run before adoption:\n%s",
			strings.Join(runtime.commands, "\n"))
	}
}