mesh-host/internal/bootstrap/bootstrap.go

// Package bootstrap brings a mesh into existence on a bare machine.
//
// **Why this is a program at all.** Until now the only complete written-down copy of the
// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is
// why every gap in it kept being found late and by accident: an install procedure that lives as a
// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is
// that procedure, made into the thing it always was.
//
// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by
// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is
// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and
// listens on nothing and that what it applies comes from a file — a property that is what makes an
// always-running root daemon auditable, and that must stay literally true. This program pulls
// images and asks a running control plane questions. Same tier, same repository, different binary.
//
// **No registry is required for the mesh's own image, and no source either.** The control plane
// exists in no registry by design, and the forge that holds its source runs on the mesh — so a
// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer
// carries the image (`internal/image`) and names it by its image id: the sha256 of its own
// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq
// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream
// `name@sha256:` references and are pulled from the internet like anything else.
package bootstrap

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
// nobody can act on and this will be run over and over by somebody getting a machine working.
type Step string

const (
	StepPreflight    Step = "preflight"
	StepLoad         Step = "load"
	StepBundle       Step = "bundle"
	StepApply        Step = "apply"
	StepVerify       Step = "verify"
	StepEnrol        Step = "enrol"
	StepRegistry     Step = "registry"
	StepPublish      Step = "publish"
	StepControlPlane Step = "control-plane"
	StepRetire       Step = "retire"
)

// Steps in the order they happen, so a failure can say "step 2 of 10".
//
// The first five make a machine; the last five make a mesh that can maintain itself. They are one
// program because they are one procedure — the whole reason the pivot exists is that steps 7 to 9
// cannot happen without steps 1 to 5, and steps 1 to 5 leave something that cannot be upgraded
// without steps 7 to 9 (novox/hq ADR 0067).
var Steps = []Step{
	StepPreflight, StepLoad, StepBundle, StepApply, StepVerify,
	StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire,
}

// Error is a failure, named by the step it happened in.
type Error struct {
	Step Step
	Err  error
}

func (e *Error) Error() string {
	at := 0
	for i, s := range Steps {
		if s == e.Step {
			at = i + 1
		}
	}
	return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err)
}

func (e *Error) Unwrap() error { return e.Err }

func failed(step Step, err error) error {
	if err == nil {
		return nil
	}
	return &Error{Step: step, Err: err}
}

// Options are the things that differ between machines.
type Options struct {
	// Template is the substrate bundle this machine's own bundle is made from.
	Template string
	// Out is where the produced bundle is written, so a person can read what was applied.
	Out string
	// State is where the host records what it has applied here — the same file `mesh-host` reads,
	// because what this raises the host must afterwards own.
	State string
	// System is which half of the host applies things. Empty means ask the machine.
	System string
	// DryRun does everything that does not change the machine.
	DryRun bool
	// Timeout bounds any single probe.
	Timeout time.Duration
	// Wait is how long something that is merely starting is given: a socket-activated container
	// runtime, a control plane opening its stores.
	Wait time.Duration

	// Node is the name this machine is known by in the mesh. Everything after the substrate names
	// it: the record, the token, the assignment, the push.
	Node string

	// Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and
	// the control plane's manifests are read from. Empty stops the installer after the substrate:
	// there is no pivot without manifests, and pretending otherwise would leave a machine that
	// looks installed and cannot upgrade itself.
	Catalogue string

	// Registry is where this mesh's own images live, as this machine reaches it. Every node will
	// pull the control plane from what this says, so on a mesh of more than one machine it must be
	// an address the others can reach.
	Registry string

	// Host is the `mesh-host` binary on this machine — the program that enrols and then holds the
	// machine to what the mesh says. The installer runs it; it does not contain it.
	Host string
	// HostService is the unit that supervises it. Started and enabled, never written: what a unit
	// says is a packaging decision, and an installer inventing one would put a file on the machine
	// that whatever installed the host will disagree with.
	HostService string
	// HostInBackground starts the host unsupervised instead, which is what the lab does and what no
	// real machine should do — it does not survive a reboot.
	HostInBackground bool
}

// pivots reports whether this run goes past the substrate.
func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" }

// Deps are the ways this program reaches outside itself. Injected so the whole of it can be
// tested without a container runtime, a network, or a machine to break — the same reason
// `internal/apply` takes a Runner (novox/hq ADR 0017).
type Deps struct {
	// Run executes a command. apply.ExecRunner in production.
	Run Runner
	// Dial reports whether a TCP address answers, for "can this machine reach the registries the
	// bundle names".
	Dial func(ctx context.Context, address string) error
	// Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own
	// registry: a container that is up is not a registry that serves, and `/v2/` is the one
	// question whose answer means it is.
	Fetch func(ctx context.Context, url string) (int, string, error)
}

// Result is what the bootstrap did, in the shape `--json` prints.
type Result struct {
	System string `json:"system"`
	DryRun bool   `json:"dry-run,omitempty"`

	// Image is the control plane's image id — what the produced bundle names it by.
	Image string `json:"image,omitempty"`
	// ImageTags is what that image was called when it was saved. Decoration, for a person.
	ImageTags []string `json:"image-tags,omitempty"`
	// ImageHeld is true when the machine already held it and nothing was loaded.
	ImageHeld bool `json:"image-already-held,omitempty"`

	// Bundle is where the produced bundle was written, and what was done to produce it.
	Bundle       string `json:"bundle,omitempty"`
	BundleWas    string `json:"bundle-replaced,omitempty"`
	BundlePlaces int    `json:"bundle-places,omitempty"`
	BundleWrote  bool   `json:"bundle-written,omitempty"`

	// Applied is how many resources the apply reported on, and whether any of them moved.
	Applied int  `json:"applied,omitempty"`
	Changed bool `json:"changed,omitempty"`

	// Running is the substrate's containers, confirmed up.
	Running []string `json:"running,omitempty"`
	// Answered is what the temporary control plane said back — not merely that it is up.
	Answered string `json:"temporary-control-plane,omitempty"`
	// Temporary is what the substrate's control plane is called, which is not what the module's is.
	Temporary string `json:"temporary-container,omitempty"`

	// Node is this machine's name in the mesh, and how it came to be enrolled and heard from.
	Node      string `json:"node,omitempty"`
	NodeAdded bool   `json:"node-record-created,omitempty"`
	Enrolled  bool   `json:"enrolled-now,omitempty"`
	Agent     string `json:"host-agent,omitempty"`

	// Registry is the mesh's own artifact store, once it answers.
	Registry         string `json:"registry,omitempty"`
	RegistryReplied  int    `json:"registry-replied,omitempty"`
	RegistryKnown    bool   `json:"registry-already-registered,omitempty"`
	RegistryRunning  string `json:"registry-container,omitempty"`
	PublishedAs      string `json:"control-plane-image,omitempty"`
	PublishedAlready bool   `json:"control-plane-image-already-published,omitempty"`

	// Permanent is the control plane as an ordinary module.
	Permanent          string   `json:"permanent-container,omitempty"`
	PermanentAnswered  string   `json:"permanent-control-plane,omitempty"`
	StoresDelivered    []string `json:"stores-delivered,omitempty"`
	TemporaryRetired   bool     `json:"temporary-retired,omitempty"`
	TemporaryWasGone   bool     `json:"temporary-was-already-gone,omitempty"`
	TemporaryRemovedAt int      `json:"resources-removed,omitempty"`

	// Stopped names why a run went no further. Empty on a run that pivoted.
	Stopped string `json:"stopped,omitempty"`
}

// Run performs the bootstrap, saying what it is doing as it goes.
//
// Every step is idempotent, and every step says whether it found something or changed it. That is
// not politeness: this program is run repeatedly while somebody gets a machine working, and a step
// that cannot tell "already done" from "just done" makes the second run indistinguishable from the
// first — which is how a person stops believing any of it.
//
// It does not retry. A pull that failed for a reason that goes away by itself is real, and the
// answer to it is to run this again: re-running is the retry, and it is one a person chooses after
// reading which step failed and why.
//
// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is
// named by the digest of its own configuration, because nothing has ever served that image and
// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine
// enrols, the registry module is installed, the carried image is pushed INTO that registry — which
// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary
// module pinned to it. The temporary one is then dropped from the bundle and the host removes it.
//
// **What makes the last part expressible is a name.** The substrate's control plane is called
// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners:
// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction
// by omission is the right end for something named "temp".
//
// Without --catalog it stops after step 5 and says so, because there are no manifests to install
// and a machine that looks installed and cannot upgrade itself is worse than one that stopped.
//
// **What an interruption leaves, at every step, and how a re-run continues.** This matters more
// here than anywhere else in the repository, because a machine left without a control plane cannot
// be fixed remotely — so no step may leave one:
//
//	1–3  nothing on the machine but a written file. Re-run: the bundle is produced again.
//	4    a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest.
//	5    everything up; something did not answer yet. Re-run: it is asked again.
//	6    a node record and possibly a spent token. Re-run: `node list` finds the record, the
//	     identity file says whether this machine enrolled, and a fresh token is issued if not.
//	7    the registry registered, assigned, maybe not applied. Re-run: registered again (an
//	     upsert), pushed again, waited for again. The temporary control plane is untouched.
//	8    the image pushed and the digest unread. Re-run: the registry is asked what it holds and
//	     the answer is the same digest; nothing is pushed twice.
//	9    the module registered and the container not yet up, OR up beside the temporary one. Both
//	     are working states: the mesh has a control plane throughout. Re-run: continues.
//	10   the bundle rewritten and the container still there. Re-run: the bundle already omits it
//	     and the apply removes it; a bundle that already omits it reports "already dropped".
//
// The only step that cannot be undone by re-running is enrolment, and that is refused rather than
// repeated: a second identity is one the mesh does not know, and the mesh believes the first.
func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) {
	if say == nil {
		say = func(string) {}
	}
	result := Result{DryRun: o.DryRun}

	// ---- 1. preflight -------------------------------------------------------------------
	say("preflight — what has to be true before anything is changed")
	template, err := Preflight(ctx, o, d, say)
	if err != nil {
		return result, failed(StepPreflight, err)
	}

	// Which half of the host applies things here. Asked of the machine and proved, because
	// `mesh-host` pins this at link time and an installer run by hand has no link time.
	sys, err := WorkOutSystem(ctx, d.Run, o.System)
	if err != nil {
		return result, failed(StepPreflight, err)
	}
	result.System = sys.Name()
	say("  system            " + sys.Name())

	// ---- 2. load ------------------------------------------------------------------------
	say("load — the control plane's image, carried in this installer")
	loaded, err := Load(ctx, d.Run, o.DryRun, say)
	if err != nil {
		return result, failed(StepLoad, err)
	}
	result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held

	// ---- 3. bundle ----------------------------------------------------------------------
	say("bundle — what this machine will be asked to be")
	rewritten, err := Rewrite(template, loaded.ID)
	if err != nil {
		return result, failed(StepBundle, err)
	}
	result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out
	result.Temporary = rewritten.TempName
	if rewritten.Renamed {
		say(fmt.Sprintf("  control plane     %s, renamed from %s",
			rewritten.TempName, rewritten.WasCalled))
		say("                    the permanent one is a module and takes the plain name; " +
			"this one is dropped at the end")
	} else {
		say("  control plane     " + rewritten.TempName + " — the template already named it that")
	}
	if rewritten.Changed {
		say(fmt.Sprintf("  its image         %s", rewritten.Now))
		say(fmt.Sprintf("  replacing         %s, named in %d place(s)",
			rewritten.Was, rewritten.Places))
	} else {
		say(fmt.Sprintf("  its image         %s — the template already named it, nothing rewritten",
			rewritten.Now))
	}
	for _, kept := range rewritten.Kept {
		say("  left alone        " + kept)
	}
	if rewritten.BrokerAddress != "" {
		// Said every time, and never changed. Every enrolment token this mesh issues will tell a
		// joining node to dial this address, and a wrong one is silent until the second node fails
		// to come back. The installer does not know this machine's address and will not invent it.
		say("  nodes will dial   " + rewritten.BrokerAddress +
			" — check this is an address other machines can reach")
	}

	if o.DryRun {
		// Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it
		// if wrong while changing nothing. The bundle has been produced and re-parsed in memory,
		// which is everything that could be checked without touching the machine; what is left is
		// loading, applying and asking the result questions, and none of those can be answered by
		// not doing them.
		say(fmt.Sprintf("  would write       %s (%d resources)", o.Out, rewritten.Resources))
		if o.pivots() {
			// Named rather than attempted. Everything from step 6 on is a conversation with a
			// control plane that a dry run has not raised, so there is nothing to ask and nothing
			// honest to report about the answers.
			say("  would then        enrol " + o.Node + ", install the registry from " +
				o.Catalogue + ", push the control plane's image into it,")
			say("                    reinstall the control plane as a module, and drop " +
				rewritten.TempName)
		}
		result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " +
			"loaded or applied"
		say("\n" + result.Stopped)
		return result, nil
	}

	if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil {
		return result, failed(StepBundle, err)
	}
	result.BundleWrote = true
	say(fmt.Sprintf("  wrote             %s (%d resources) — read it, this is what is applied",
		o.Out, rewritten.Resources))

	// ---- 4. apply -----------------------------------------------------------------------
	say("apply — raising the substrate")
	report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say)
	result.Applied, result.Changed = len(report.Outcomes), report.Changed()
	if err != nil {
		return result, failed(StepApply, err)
	}
	if report.Changed() {
		say(fmt.Sprintf("  applied           %d resource(s)", len(report.Outcomes)))
	} else {
		say(fmt.Sprintf("  already matches   %d resource(s) checked, nothing moved",
			len(report.Outcomes)))
	}

	// ---- 5. verify ----------------------------------------------------------------------
	say("verify — the substrate is up, and the control plane replies")
	verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say)
	result.Running, result.Answered = verified.Running, verified.Answered
	if err != nil {
		return result, failed(StepVerify, err)
	}

	if !o.pivots() {
		// Stopped, and said plainly. What has been raised works and cannot be upgraded: its
		// control plane is named by an image id, which no registry serves, so nothing can ever
		// replace it with a newer one. That is the whole of what the pivot fixes, and it needs
		// manifests, and manifests come from a checkout somebody has to point this at.
		result.Stopped = "no --catalog was given, so this stopped at the substrate. " +
			"The control plane is named by the digest of its own configuration and no registry " +
			"serves it, so this mesh cannot yet upgrade itself. Run again with " +
			"--catalog <a checkout of the mesh's catalogue> to finish the pivot; every step " +
			"above will say it is already done"
		say("\nthis machine is a mesh of one node, with nothing joined to it yet.")
		say(result.Stopped)
		return result, nil
	}

	// ---- 6. enrol -------------------------------------------------------------------------
	//
	// From here on the mesh is being told things, and the way to tell it anything is to run its
	// own binary inside its own container. `temporary` is the substrate's control plane; the
	// module's is a different container with a different name and does not exist yet.
	temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout}

	say("enrol — this machine joins the mesh it is running")
	enrolled, err := Enrol(ctx, o, sys, temporary, say)
	result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined
	result.Agent = enrolled.Agent
	if err != nil {
		return result, failed(StepEnrol, err)
	}

	// ---- 7. registry ----------------------------------------------------------------------
	say("registry — somewhere for this mesh to keep its own images")
	registry, err := InstallRegistry(ctx, o, d, temporary, say)
	result.Registry, result.RegistryRunning = registry.Address, registry.Container
	result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
	if err != nil {
		return result, failed(StepRegistry, err)
	}

	// ---- 8. publish -----------------------------------------------------------------------
	say("publish — the control plane's image gets its first manifest digest")
	published, err := PublishControlPlane(ctx, o, d, loaded.ID, say)
	result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
	if err != nil {
		return result, failed(StepPublish, err)
	}

	// ---- 9. control plane -----------------------------------------------------------------
	say("control plane — installed as an ordinary module, pinned to that digest")
	permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration,
		published.Reference, say)
	result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered
	result.StoresDelivered = permanent.Delivered
	if err != nil {
		return result, failed(StepControlPlane, err)
	}

	// ---- 10. retire -----------------------------------------------------------------------
	say("retire — the temporary control plane is dropped from the bundle")
	retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say)
	result.TemporaryRetired = retired.Gone || retired.Already
	result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed
	if err != nil {
		return result, failed(StepRetire, err)
	}

	say("\nthis machine is a mesh of one node, and the control plane it runs is a module " +
		"pinned to an image its own registry serves.")
	say("what remains is somebody else's: adding nodes, and assigning what they should run.")

	return result, nil
}