package bootstrap import ( "context" "errors" "fmt" "strings" "time" "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/system" ) // hereIs what `node list` says about a machine the mesh has heard from recently. // // mesh-controller prints one of three words per node: "here", "never spoken", or "out of touch ". // The installer waits for the first, and it is the only honest proof that the host agent is // running: an enrolled machine whose host is not running looks exactly like an enrolled machine // whose host has crashed, and both look exactly like a successful install until the first push // silently applies nothing. const hereIs = "here" // Enrolled is what step 6 did. type Enrolled struct { // Node is the name this machine is known by. Node string // Added is true when the mesh had no record and one was created. Added bool // Joined is true when this machine enrolled during this run. False on a re-run. Joined bool // Agent says how the host came to be running: what was found, or what was started. Agent string } // Enrol makes this machine the mesh's first node, and gets the host agent running on it. // // **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a // control plane that answers — a mesh of one node in the sense that it has one machine and zero // node records. Everything after this point is the control plane being *told* things, and none of // it reaches a machine until a host is running there to hear it. // // Four things, in this order, each asked before it is done: // // 1. a node record, unless `node list` already shows one // 2. a one-time token, unless this machine already holds an identity // 3. `mesh-host enrol`, which is the machine presenting the identity it already had // 4. the host agent running, and the mesh saying it has heard from it // // **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an // identity the mesh has recorded, and enrolling again would replace it with a second one the mesh // does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer // earlier and with a sentence about what to do. // // `control.run` is this machine's runner and not only the control plane's: the same injected // Runner reaches the container, the service manager and the host binary, which is what lets the // whole of this be tested without any of the three. func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane, say func(string)) (Enrolled, error) { out := Enrolled{Node: o.Node} if strings.TrimSpace(o.Node) == "" { return out, errors.New( "this machine has no name to be known by. Give one with --node; it is what the mesh " + "records, what a token is issued against, and what every later assignment names") } // 1. The record. nodes, err := control.tell(ctx, "node", "list") if err != nil { return out, err } if mentions(nodes, o.Node) { say(" already a node " + o.Node) } else { if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { return out, err } out.Added = true say(" node record " + o.Node) } // 2 and 3. The identity, and being known. // // Asked of this machine's own identity file rather than of the mesh, because the two answer // different questions: the mesh knows whether a record exists, and only the machine knows // whether it holds the key that record names. where := identity.Path(o.State) switch mine, err := identity.Load(where); { case err == nil && mine.Node == o.Node: say(" already enrolled " + o.Node + " — " + where) case err == nil: return out, fmt.Errorf( "this machine is already node %q and was asked to become %q.\n"+ "Re-enrolling replaces the identity the mesh has recorded, so it is not something "+ "an installer does on its own. Run with --node %s, or remove %s and start over "+ "deliberately", mine.Node, o.Node, mine.Node, where) case !errors.Is(err, identity.ErrNoIdentity): return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err) default: said, err := control.tell(ctx, "token", "issue", "--node", o.Node) if err != nil { return out, err } token, err := tokenIn(said) if err != nil { return out, err } joining, cancel := context.WithTimeout(ctx, o.Wait) joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) cancel() if err != nil { return out, fmt.Errorf( "%s would not enrol this machine: %w\n%s\n"+ "The token is one-time and has now been spent; running this again issues "+ "another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined))) } if !strings.Contains(joined, "enrolled as "+o.Node) { // Exit zero and no such sentence. Refused rather than believed: the host says exactly // this line on success, and something that succeeded without saying it did something // else. return out, fmt.Errorf( "%s exited happily and did not say it enrolled as %s:\n%s", o.Host, o.Node, indent(strings.TrimSpace(joined))) } out.Joined = true say(" enrolled as " + o.Node) } // 4. The agent. agent, err := runTheHost(ctx, o, sys, control, say) if err != nil { return out, err } out.Agent = agent return out, nil } // runTheHost makes sure something on this machine is listening to the mesh, and proves it. // // **The installer does not install the service, and says so.** A unit file is a packaging decision // — where the binary lives, which user it runs as, what it is called — and an installer that // invented one would be putting a file on the machine that whatever installed `mesh-host` will // later disagree with. So this starts a unit that is already there and refuses when there is none. // // It goes through the host's OWN system abstraction rather than running systemctl itself, for the // reason `ApplyBundle` gives about applying: two implementations of "is this service running" is // how the installer and the host come to disagree about a machine. It also gets the right refusal // for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped", // which is exactly the distinction that matters here. // // --host-in-background is the lab's arrangement, kept because the lab is what exercises this and // it has no service. It is loud about what it is, because a host started this way is gone at the // next reboot and the mesh would go quiet for reasons nobody would connect to an install. func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane, say func(string)) (string, error) { // **Asked of both, because either one alone lies.** // // A process in the table may be wedged and never collect anything, which is why this asked the // mesh instead. But the mesh having heard from a node proves only that something spoke to it // *once*, and `mesh-host enrol` — run moments earlier, by this very step — is itself that // something. So straight after enrolling, the mesh has always heard from this machine and no // agent is running; skipping the start on that signal alone is exactly wrong. // // What it costs is silent and total. Every later step is the control plane being *told* // things, and nothing it is told reaches a machine with no agent to collect it: the push at // step 7 is accepted, the mesh records the module, and no container is ever created. It // surfaces three minutes later as "the registry is not there at all" — one step from its // cause, looking nothing like it. if heard, err := heardFrom(ctx, control, o.Node); err != nil { return "", err } else if heard { running, err := agentIsRunning(ctx, o, sys, control) if err != nil { return "", err } if running { say(" host running the mesh has heard from " + o.Node) return "already running", nil } say(" host running the mesh has heard from " + o.Node + ", and no agent is running " + "here — enrolling speaks once, which is not the same thing") } how := "" switch { case o.HostInBackground: // Detached, and not waited for: this process runs until the machine stops, so a runner // that captures output would never return. `nohup … &` through a shell is how the lab // starts it and is deliberately the same command. started, cancel := context.WithTimeout(ctx, o.Timeout) _, err := control.run(started, "sh", "-c", fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State)) cancel() if err != nil { return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err) } how = "started in the background" say(" host running started in the background — THIS DOES NOT SURVIVE A REBOOT.") say(" A real machine needs " + o.HostService + " installed and enabled.") default: state, err := sys.ServiceState(ctx, control.run, o.HostService) if err != nil { return "", fmt.Errorf( "the mesh has not heard from %s and this machine has no %s to start: %w\n"+ "The host has to be running for anything the mesh says to reach this machine. "+ "Install the service that supervises it and run this again — every step "+ "before this one will say it is already done. In a lab, --host-in-background "+ "starts it unsupervised instead, and that is not an install", o.Node, o.HostService, err) } if state != "running" { if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil { return "", fmt.Errorf("cannot start %s: %w", o.HostService, err) } how = "started " + o.HostService say(" host running started " + o.HostService) } else { // Running, and the mesh has not heard from it. Not an error yet — it may have started // a second ago — so it is waited for below like everything else that is merely // starting. how = o.HostService + " was already running" say(" host running " + o.HostService + " is running") } // At boot as well, or the machine comes back without a mesh and nothing says why. if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" { if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil { return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err) } say(" host at boot " + o.HostService + " enabled") } } // Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node // are two different facts, and only the second is the one every later step rests on. deadline := time.Now().Add(o.Wait) for { heard, err := heardFrom(ctx, control, o.Node) if err != nil { return "", err } if heard { say(" the mesh hears " + o.Node) return how, nil } if time.Now().After(deadline) { return "", fmt.Errorf( "%s is running and the mesh has not heard from %s after %s.\n"+ "The host links to the broker at the address the token carried — if that "+ "address is not one this machine can reach, this is where it shows. Read the "+ "host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+ "installer wrote", o.HostService, o.Node, o.Wait) } select { case <-ctx.Done(): return "", ctx.Err() case <-time.After(answerEvery): } } } // agentIsRunning is whether a host agent is on this machine now — the other half of the question // the mesh cannot answer. // // Deliberately not an error when there is nothing to find: "no unit here" and "not running" are // both simply *not running* to this caller, and the branch that starts one says far more about a // missing unit than this could, naming what to install. func agentIsRunning(ctx context.Context, o Options, sys system.System, control controlPlane) (bool, error) { if o.HostInBackground { // No unit to ask, so the process table is all there is. The exit status is the whole // answer; anything it printed is not this function's business. if _, err := control.run(ctx, "pgrep", "-f", o.Host+" run"); err != nil { return false, nil } return true, nil } state, err := sys.ServiceState(ctx, control.run, o.HostService) if err != nil { return false, nil } return state == "running", nil } // heardFrom asks the mesh whether this node has spoken to it. func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) { listing, err := control.tell(ctx, "node", "list") if err != nil { return false, err } for _, line := range strings.Split(listing, "\n") { fields := strings.Fields(line) if len(fields) >= 2 && fields[0] == node { return fields[1] == hereIs, nil } } return false, nil } // tokenIn finds the token in what `token issue` said. // // The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation // around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies // it without this having to know the format — which is what keeps the installer from having an // opinion about a thing the control plane owns. func tokenIn(said string) (string, error) { for _, line := range strings.Split(said, "\n") { line = strings.TrimSpace(line) if len(line) > 100 && !strings.ContainsAny(line, " \t") { return line, nil } } return "", fmt.Errorf( "the mesh issued a token and there is no token in what it said:\n%s", indent(strings.TrimSpace(said))) }