mesh-host/internal/bootstrap/enrol.go

package bootstrap

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/identity"
	"github.com/novox/mesh-host/internal/system"
)

// hereIs what `node list` says about a machine the mesh has heard from recently.
//
// mesh-controller prints one of three words per node: "here", "never spoken", or "out of touch <age>".
// The installer waits for the first, and it is the only honest proof that the host agent is
// running: an enrolled machine whose host is not running looks exactly like an enrolled machine
// whose host has crashed, and both look exactly like a successful install until the first push
// silently applies nothing.
const hereIs = "here"

// Enrolled is what step 6 did.
type Enrolled struct {
	// Node is the name this machine is known by.
	Node string
	// Added is true when the mesh had no record and one was created.
	Added bool
	// Joined is true when this machine enrolled during this run. False on a re-run.
	Joined bool
	// Agent says how the host came to be running: what was found, or what was started.
	Agent string
}

// Enrol makes this machine the mesh's first node, and gets the host agent running on it.
//
// **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a
// control plane that answers — a mesh of one node in the sense that it has one machine and zero
// node records. Everything after this point is the control plane being *told* things, and none of
// it reaches a machine until a host is running there to hear it.
//
// Four things, in this order, each asked before it is done:
//
//  1. a node record, unless `node list` already shows one
//  2. a one-time token, unless this machine already holds an identity
//  3. `mesh-host enrol`, which is the machine presenting the identity it already had
//  4. the host agent running, and the mesh saying it has heard from it
//
// **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an
// identity the mesh has recorded, and enrolling again would replace it with a second one the mesh
// does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer
// earlier and with a sentence about what to do.
//
// `control.run` is this machine's runner and not only the control plane's: the same injected
// Runner reaches the container, the service manager and the host binary, which is what lets the
// whole of this be tested without any of the three.
func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane,
	say func(string)) (Enrolled, error) {

	out := Enrolled{Node: o.Node}
	if strings.TrimSpace(o.Node) == "" {
		return out, errors.New(
			"this machine has no name to be known by. Give one with --node; it is what the mesh " +
				"records, what a token is issued against, and what every later assignment names")
	}

	// 1. The record.
	nodes, err := control.tell(ctx, "node", "list")
	if err != nil {
		return out, err
	}
	if mentions(nodes, o.Node) {
		say("  already a node    " + o.Node)
	} else {
		if _, err := control.tell(ctx, "node", "add", o.Node); err != nil {
			return out, err
		}
		out.Added = true
		say("  node record       " + o.Node)
	}

	// 2 and 3. The identity, and being known.
	//
	// Asked of this machine's own identity file rather than of the mesh, because the two answer
	// different questions: the mesh knows whether a record exists, and only the machine knows
	// whether it holds the key that record names.
	where := identity.Path(o.State)
	switch mine, err := identity.Load(where); {
	case err == nil && mine.Node == o.Node:
		say("  already enrolled  " + o.Node + " — " + where)
	case err == nil:
		return out, fmt.Errorf(
			"this machine is already node %q and was asked to become %q.\n"+
				"Re-enrolling replaces the identity the mesh has recorded, so it is not something "+
				"an installer does on its own. Run with --node %s, or remove %s and start over "+
				"deliberately", mine.Node, o.Node, mine.Node, where)
	case !errors.Is(err, identity.ErrNoIdentity):
		return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err)
	default:
		said, err := control.tell(ctx, "token", "issue", "--node", o.Node)
		if err != nil {
			return out, err
		}
		token, err := tokenIn(said)
		if err != nil {
			return out, err
		}
		joining, cancel := context.WithTimeout(ctx, o.Wait)
		joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State)
		cancel()
		if err != nil {
			return out, fmt.Errorf(
				"%s would not enrol this machine: %w\n%s\n"+
					"The token is one-time and may have been spent; running this again issues "+
					"another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined)))
		}
		if !strings.Contains(joined, "enrolled as "+o.Node) {
			// Exit zero and no such sentence. Refused rather than believed: the host says exactly
			// this line on success, and something that succeeded without saying it did something
			// else.
			return out, fmt.Errorf(
				"%s exited happily and did not say it enrolled as %s:\n%s",
				o.Host, o.Node, indent(strings.TrimSpace(joined)))
		}
		out.Joined = true
		say("  enrolled as       " + o.Node)
	}

	// 4. The agent.
	agent, err := runTheHost(ctx, o, sys, control, say)
	if err != nil {
		return out, err
	}
	out.Agent = agent
	return out, nil
}

// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
//
// **The installer does not install the service, and says so.** A unit file is a packaging decision
// — where the binary lives, which user it runs as, what it is called — and an installer that
// invented one would be putting a file on the machine that whatever installed `mesh-host` will
// later disagree with. So this starts a unit that is already there and refuses when there is none.
//
// It goes through the host's OWN system abstraction rather than running systemctl itself, for the
// reason `ApplyBundle` gives about applying: two implementations of "is this service running" is
// how the installer and the host come to disagree about a machine. It also gets the right refusal
// for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped",
// which is exactly the distinction that matters here.
//
// --host-in-background is the lab's arrangement, kept because the lab is what exercises this and
// it has no service. It is loud about what it is, because a host started this way is gone at the
// next reboot and the mesh would go quiet for reasons nobody would connect to an install.
func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane,
	say func(string)) (string, error) {

	// **Asked of both, because either one alone lies.**
	//
	// A process in the table may be wedged and never collect anything, which is why this asked the
	// mesh instead. But the mesh having heard from a node proves only that something spoke to it
	// *once*, and `mesh-host enrol` — run moments earlier, by this very step — is itself that
	// something. So straight after enrolling, the mesh has always heard from this machine and no
	// agent is running; skipping the start on that signal alone is exactly wrong.
	//
	// What it costs is silent and total. Every later step is the control plane being *told*
	// things, and nothing it is told reaches a machine with no agent to collect it: the push at
	// step 7 is accepted, the mesh records the module, and no container is ever created. It
	// surfaces three minutes later as "the registry is not there at all" — one step from its
	// cause, looking nothing like it.
	if heard, err := heardFrom(ctx, control, o.Node); err != nil {
		return "", err
	} else if heard {
		running, err := agentIsRunning(ctx, o, sys, control)
		if err != nil {
			return "", err
		}
		if running {
			say("  host running      the mesh has heard from " + o.Node)
			return "already running", nil
		}
		say("  host running      the mesh has heard from " + o.Node + ", and no agent is running " +
			"here — enrolling speaks once, which is not the same thing")
	}

	how := ""
	switch {
	case o.HostInBackground:
		// Detached, and not waited for: this process runs until the machine stops, so a runner
		// that captures output would never return. `nohup … &` through a shell is how the lab
		// starts it and is deliberately the same command.
		started, cancel := context.WithTimeout(ctx, o.Timeout)
		_, err := control.run(started, "sh", "-c",
			fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State))
		cancel()
		if err != nil {
			return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err)
		}
		how = "started in the background"
		say("  host running      started in the background — THIS DOES NOT SURVIVE A REBOOT.")
		say("                    A real machine needs " + o.HostService + " installed and enabled.")

	default:
		state, err := sys.ServiceState(ctx, control.run, o.HostService)
		if err != nil {
			return "", fmt.Errorf(
				"the mesh has not heard from %s and this machine has no %s to start: %w\n"+
					"The host has to be running for anything the mesh says to reach this machine. "+
					"Install the service that supervises it and run this again — every step "+
					"before this one will say it is already done. In a lab, --host-in-background "+
					"starts it unsupervised instead, and that is not an install",
				o.Node, o.HostService, err)
		}
		if state != "running" {
			if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil {
				return "", fmt.Errorf("cannot start %s: %w", o.HostService, err)
			}
			how = "started " + o.HostService
			say("  host running      started " + o.HostService)
		} else {
			// Running, and the mesh has not heard from it. Not an error yet — it may have started
			// a second ago — so it is waited for below like everything else that is merely
			// starting.
			how = o.HostService + " was already running"
			say("  host running      " + o.HostService + " is running")
		}
		// At boot as well, or the machine comes back without a mesh and nothing says why.
		if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" {
			if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil {
				return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err)
			}
			say("  host at boot      " + o.HostService + " enabled")
		}
	}

	// Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node
	// are two different facts, and only the second is the one every later step rests on.
	deadline := time.Now().Add(o.Wait)
	for {
		heard, err := heardFrom(ctx, control, o.Node)
		if err != nil {
			return "", err
		}
		if heard {
			say("  the mesh hears    " + o.Node)
			return how, nil
		}
		if time.Now().After(deadline) {
			return "", fmt.Errorf(
				"%s is running and the mesh has not heard from %s after %s.\n"+
					"The host links to the broker at the address the token carried — if that "+
					"address is not one this machine can reach, this is where it shows. Read the "+
					"host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+
					"installer wrote", o.HostService, o.Node, o.Wait)
		}
		select {
		case <-ctx.Done():
			return "", ctx.Err()
		case <-time.After(answerEvery):
		}
	}
}

// agentIsRunning is whether a host agent is on this machine now — the other half of the question
// the mesh cannot answer.
//
// Deliberately not an error when there is nothing to find: "no unit here" and "not running" are
// both simply *not running* to this caller, and the branch that starts one says far more about a
// missing unit than this could, naming what to install.
func agentIsRunning(ctx context.Context, o Options, sys system.System, control controlPlane) (bool, error) {
	if o.HostInBackground {
		// No unit to ask, so the process table is all there is. The exit status is the whole
		// answer; anything it printed is not this function's business.
		if _, err := control.run(ctx, "pgrep", "-f", o.Host+" run"); err != nil {
			return false, nil
		}
		return true, nil
	}
	state, err := sys.ServiceState(ctx, control.run, o.HostService)
	if err != nil {
		return false, nil
	}
	return state == "running", nil
}

// heardFrom asks the mesh whether this node has spoken to it.
func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) {
	listing, err := control.tell(ctx, "node", "list")
	if err != nil {
		return false, err
	}
	for _, line := range strings.Split(listing, "\n") {
		fields := strings.Fields(line)
		if len(fields) >= 2 && fields[0] == node {
			return fields[1] == hereIs, nil
		}
	}
	return false, nil
}

// tokenIn finds the token in what `token issue` said.
//
// The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation
// around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies
// it without this having to know the format — which is what keeps the installer from having an
// opinion about a thing the control plane owns.
func tokenIn(said string) (string, error) {
	for _, line := range strings.Split(said, "\n") {
		line = strings.TrimSpace(line)
		if len(line) > 100 && !strings.ContainsAny(line, " \t") {
			return line, nil
		}
	}
	return "", fmt.Errorf(
		"the mesh issued a token and there is no token in what it said:\n%s",
		indent(strings.TrimSpace(said)))
}