mesh-host/internal/bootstrap/bootstrap.go

// Package bootstrap brings a mesh into existence on a bare machine.
//
// **Why this is a program at all.** Until now the only complete written-down copy of the
// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is
// why every gap in it kept being found late and by accident: an install procedure that lives as a
// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is
// that procedure, made into the thing it always was.
//
// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by
// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is
// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and
// listens on nothing and that what it applies comes from a file — a property that is what makes an
// always-running root daemon auditable, and that must stay literally true. This program pulls
// images and asks a running control plane questions. Same tier, same repository, different binary.
//
// **No registry is required for the mesh's own image, and no source either.** The control plane
// exists in no registry by design, and the forge that holds its source runs on the mesh — so a
// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer
// carries the image (`internal/image`) and names it by its image id: the sha256 of its own
// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq
// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream
// `name@sha256:` references and are pulled from the internet like anything else.
//
// **And that id is read back from the machine, never predicted from the archive.** The digest of a
// configuration is not portable: a runtime rewrites the configuration as it loads, so the same
// bytes are held under a different name on the machine that receives them than on the one that
// saved them. The archive is identified by its TAG, which does survive the transfer, and the id
// the bundle names is whatever the runtime answers for that tag afterwards. See Load, which
// carries the measurement.
package bootstrap

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
// nobody can act on and this will be run over and over by somebody getting a machine working.
type Step string

const (
	StepPreflight    Step = "preflight"
	StepLoad         Step = "load"
	StepBuild        Step = "build"
	StepBundle       Step = "bundle"
	StepApply        Step = "apply"
	StepVerify       Step = "verify"
	StepEnrol        Step = "enrol"
	StepRegistry     Step = "registry"
	StepPublish      Step = "publish"
	StepControlPlane Step = "control-plane"
	StepRetire       Step = "retire"
	StepBuilder      Step = "builder"
	StepPackages     Step = "packages"
	StepSDK          Step = "sdk"
	StepBase         Step = "base"
	StepStore        Step = "store"
	StepCatalogue    Step = "catalogue"
	StepNetwork      Step = "network"
	StepFilter       Step = "filter"
	StepExtras       Step = "extras"
)

// Steps in the order they happen, so a failure can say "step 2 of 11".
//
// The first six make a machine; the last five make a mesh that can maintain itself. They are one
// program because they are one procedure — the whole reason the pivot exists is that steps 8 to 10
// cannot happen without steps 1 to 6, and steps 1 to 6 leave something that cannot be upgraded
// without steps 8 to 10 (novox/hq ADR 0067).
//
// **Build sits between load and bundle**, because the bundle has to name an image and that image
// no longer arrives finished. The installer carries the builder, loads it, and uses it to produce
// the control plane from source (novox/hq ADR 0073) — so what the bundle names is something this
// mesh made, out of a repository and a commit it can name, and can therefore make again.
var Steps = []Step{
	StepPreflight, StepLoad, StepBuild, StepBundle, StepApply, StepVerify,
	StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, StepBuilder,
	StepPackages, StepSDK,
	// Phase two. The twelve above make a mesh that RUNS; these make one that WORKS — able to
	// build, to say what it holds, on its network, filtering. They used to be things somebody
	// typed afterwards, which is how they went missing without anything complaining.
	StepBase, StepStore, StepCatalogue, StepNetwork, StepFilter, StepExtras,
}

// Error is a failure, named by the step it happened in.
type Error struct {
	Step Step
	Err  error
}

func (e *Error) Error() string {
	at := 0
	for i, s := range Steps {
		if s == e.Step {
			at = i + 1
		}
	}
	return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err)
}

func (e *Error) Unwrap() error { return e.Err }

func failed(step Step, err error) error {
	if err == nil {
		return nil
	}
	return &Error{Step: step, Err: err}
}

// Options are the things that differ between machines.
type Options struct {
	// Template is the substrate bundle this machine's own bundle is made from.
	Template string
	// Out is where the produced bundle is written, so a person can read what was applied.
	Out string
	// State is where the host records what it has applied here — the same file `mesh-host` reads,
	// because what this raises the host must afterwards own.
	State string
	// System is which half of the host applies things. Empty means ask the machine.
	System string
	// DryRun does everything that does not change the machine.
	DryRun bool
	// Timeout bounds any single probe.
	Timeout time.Duration
	// Wait is how long something that is merely starting is given: a socket-activated container
	// runtime, a control plane opening its stores.
	Wait time.Duration

	// Node is the name this machine is known by in the mesh. Everything after the substrate names
	// it: the record, the token, the assignment, the push.
	Node string

	// Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and
	// the control plane's manifests are read from. Empty stops the installer after the substrate:
	// there is no pivot without manifests, and pretending otherwise would leave a machine that
	// looks installed and cannot upgrade itself.
	Catalogue string

	// Source is where the control plane is built from — a repository on a mesh that already
	// exists, and a commit. The installer carries the builder rather than a finished control
	// plane (novox/hq ADR 0073), so this is what it is told to make.
	Source Source

	// Registry is where this mesh's own images live, as this machine reaches it. Every node will
	// pull the control plane from what this says, so on a mesh of more than one machine it must be
	// an address the others can reach.
	Registry string

	// Host is the `mesh-host` binary on this machine — the program that enrols and then holds the
	// machine to what the mesh says. The installer runs it; it does not contain it.
	Host string
	// HostService is the unit that supervises it. Started and enabled, never written: what a unit
	// says is a packaging decision, and an installer inventing one would put a file on the machine
	// that whatever installed the host will disagree with.
	HostService string
	// HostInBackground starts the host unsupervised instead, which is what the lab does and what no
	// real machine should do — it does not survive a reboot.
	HostInBackground bool

	// ---- phase two — a mesh that runs becomes a mesh that works ----

	// ToolsSource is where the shared base is built from. Everything with code of its own
	// compiles against it, so it is the first thing the mesh builds for itself.
	ToolsSource Source
	// CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue
	// CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it.
	CatalogSource Source
	// SDKSource is where the mesh's shared library is built from. It is published to the package
	// registry before the base is built, because the base resolves it by version (novox/hq ADR 0076).
	SDKSource Source
	// Site is where this machine sits, for the private network's placement.
	Site string
	// Answers are the choices, answered by flag: name → answer. What a terminal would be asked.
	Answers map[string]string
	// Prompt asks a person one choice. Nil is an unattended run: flags and defaults answer, and a
	// required choice nothing answered is a refusal rather than a guess.
	Prompt func(Choice) (string, error)
	// Extras are catalogue modules beyond the floor, asked for by name.
	Extras []string
}

// pivots reports whether this run goes past the substrate.
func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" }

// Deps are the ways this program reaches outside itself. Injected so the whole of it can be
// tested without a container runtime, a network, or a machine to break — the same reason
// `internal/apply` takes a Runner (novox/hq ADR 0017).
type Deps struct {
	// Run executes a command. apply.ExecRunner in production.
	Run Runner
	// Dial reports whether a TCP address answers, for "can this machine reach the registries the
	// bundle names".
	Dial func(ctx context.Context, address string) error
	// Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own
	// registry: a container that is up is not a registry that serves, and `/v2/` is the one
	// question whose answer means it is.
	Fetch func(ctx context.Context, url string) (int, string, error)
}

// Result is what the bootstrap did, in the shape `--json` prints.
type Result struct {
	System string `json:"system"`
	DryRun bool   `json:"dry-run,omitempty"`

	// Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and
	// what the produced bundle names it by.
	Image string `json:"image,omitempty"`
	// ImageArchive is what the carried archive calls the same image. Reported because it is
	// routinely a DIFFERENT id: a runtime rewrites an image's configuration as it loads, and an id
	// is that configuration's digest. Never what the bundle names.
	ImageArchive string `json:"image-in-archive,omitempty"`
	// ImageTag is the name the runtime was asked by, which is how the id above was obtained.
	ImageTag string `json:"image-tag,omitempty"`
	// ImageTags is everything the image was called when it was saved.
	ImageTags []string `json:"image-tags,omitempty"`
	// ImageHeld is true when the machine already held it and nothing was loaded.
	ImageHeld bool `json:"image-already-held,omitempty"`
	// BuilderPublished is where the builder's image ended up, and BuilderInstalled whether it is
	// a module on this machine. A mesh without them runs and cannot produce.
	BuilderPublished string `json:"builder-image,omitempty"`
	BuilderInstalled bool   `json:"builder-installed,omitempty"`
	BuilderAccount   bool   `json:"builder-account-issued,omitempty"`

	// Built is what the genesis build produced, and BuiltFrom is the commit it actually built.
	//
	// Reported because they are the difference between a mesh that can rebuild its control plane
	// and one that cannot: a machine holding these can be asked for the same thing again and get
	// the same thing back.
	Built     string `json:"built,omitempty"`
	BuiltFrom string `json:"built-from,omitempty"`

	// ImagePredicted is true when Image is the archive's id because nothing was loaded — a dry run
	// only, and the reason a dry run does not claim to know what would be applied.
	ImagePredicted bool `json:"image-id-is-a-prediction,omitempty"`

	// Bundle is where the produced bundle was written, and what was done to produce it.
	Bundle       string `json:"bundle,omitempty"`
	BundleWas    string `json:"bundle-replaced,omitempty"`
	BundlePlaces int    `json:"bundle-places,omitempty"`
	BundleWrote  bool   `json:"bundle-written,omitempty"`

	// Applied is how many resources the apply reported on, and whether any of them moved.
	Applied int  `json:"applied,omitempty"`
	Changed bool `json:"changed,omitempty"`

	// Running is the substrate's containers, confirmed up.
	Running []string `json:"running,omitempty"`
	// Answered is what the temporary control plane said back — not merely that it is up.
	Answered string `json:"temporary-control-plane,omitempty"`
	// Temporary is what the substrate's control plane is called, which is not what the module's is.
	Temporary string `json:"temporary-container,omitempty"`

	// Node is this machine's name in the mesh, and how it came to be enrolled and heard from.
	Node      string `json:"node,omitempty"`
	NodeAdded bool   `json:"node-record-created,omitempty"`
	Enrolled  bool   `json:"enrolled-now,omitempty"`
	Agent     string `json:"host-agent,omitempty"`

	// Registry is the mesh's own artifact store, once it answers.
	Registry         string `json:"registry,omitempty"`
	RegistryReplied  int    `json:"registry-replied,omitempty"`
	RegistryKnown    bool   `json:"registry-already-registered,omitempty"`
	RegistryRunning  string `json:"registry-container,omitempty"`
	PublishedAs      string `json:"control-plane-image,omitempty"`
	PublishedAlready bool   `json:"control-plane-image-already-published,omitempty"`

	// Permanent is the control plane as an ordinary module.
	Permanent          string   `json:"permanent-container,omitempty"`
	PermanentAnswered  string   `json:"permanent-control-plane,omitempty"`
	StoresDelivered    []string `json:"stores-delivered,omitempty"`
	TemporaryRetired   bool     `json:"temporary-retired,omitempty"`
	TemporaryWasGone   bool     `json:"temporary-was-already-gone,omitempty"`
	TemporaryRemovedAt int      `json:"resources-removed,omitempty"`

	// Stopped names why a run went no further. Empty on a run that pivoted.
	Stopped string `json:"stopped,omitempty"`
}

// Run performs the bootstrap, saying what it is doing as it goes.
//
// Every step is idempotent, and every step says whether it found something or changed it. That is
// not politeness: this program is run repeatedly while somebody gets a machine working, and a step
// that cannot tell "already done" from "just done" makes the second run indistinguishable from the
// first — which is how a person stops believing any of it.
//
// It does not retry. A pull that failed for a reason that goes away by itself is real, and the
// answer to it is to run this again: re-running is the retry, and it is one a person chooses after
// reading which step failed and why.
//
// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is
// named by the digest of its own configuration, because nothing has ever served that image and
// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine
// enrols, the registry module is installed, the carried image is pushed INTO that registry — which
// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary
// module pinned to it. The temporary one is then dropped from the bundle and the host removes it.
//
// **What makes the last part expressible is a name.** The substrate's control plane is called
// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners:
// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction
// by omission is the right end for something named "temp".
//
// Without --catalog it stops after step 5 and says so, because there are no manifests to install
// and a machine that looks installed and cannot upgrade itself is worse than one that stopped.
//
// **What an interruption leaves, at every step, and how a re-run continues.** This matters more
// here than anywhere else in the repository, because a machine left without a control plane cannot
// be fixed remotely — so no step may leave one:
//
//	1–3  nothing on the machine but a written file. Re-run: the bundle is produced again.
//	4    a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest.
//	5    everything up; something did not answer yet. Re-run: it is asked again.
//	6    a node record and possibly a spent token. Re-run: `node list` finds the record, the
//	     identity file says whether this machine enrolled, and a fresh token is issued if not.
//	7    the registry registered, assigned, maybe not applied. Re-run: registered again (an
//	     upsert), pushed again, waited for again. The temporary control plane is untouched.
//	8    the image pushed and the digest unread. Re-run: the registry is asked what it holds and
//	     the answer is the same digest; nothing is pushed twice.
//	9    the module registered and the container not yet up, OR up beside the temporary one. Both
//	     are working states: the mesh has a control plane throughout. Re-run: continues.
//	10   the bundle rewritten and the container still there. Re-run: the bundle already omits it
//	     and the apply removes it; a bundle that already omits it reports "already dropped".
//
// The only step that cannot be undone by re-running is enrolment, and that is refused rather than
// repeated: a second identity is one the mesh does not know, and the mesh believes the first.
func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) {
	if say == nil {
		say = func(string) {}
	}
	result := Result{DryRun: o.DryRun}

	// ---- 1. preflight -------------------------------------------------------------------
	say("preflight — what has to be true before anything is changed")
	template, err := Preflight(ctx, o, d, say)
	if err != nil {
		return result, failed(StepPreflight, err)
	}

	// Which half of the host applies things here. Asked of the machine and proved, because
	// `mesh-host` pins this at link time and an installer run by hand has no link time.
	sys, err := WorkOutSystem(ctx, d.Run, o.System)
	if err != nil {
		return result, failed(StepPreflight, err)
	}
	result.System = sys.Name()
	say("  system            " + sys.Name())

	// ---- 2. load ------------------------------------------------------------------------
	say("load — the builder's image, carried in this installer")
	loaded, err := Load(ctx, d.Run, o.DryRun, say)
	if err != nil {
		return result, failed(StepLoad, err)
	}
	// These describe the CARRIED image, which is the builder. What the control plane ends up
	// being is reported separately, by the build below.
	result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held
	result.ImageArchive, result.ImageTag = loaded.Archive, loaded.Tag
	result.ImagePredicted = loaded.Predicted

	// ---- 3. build -----------------------------------------------------------------------
	say("build — the control plane, from its own repository and a commit")
	built, err := BuildControlPlane(ctx, d.Run, loaded.Tag, o.Source, o.DryRun, say)
	if err != nil {
		return result, failed(StepBuild, err)
	}
	result.Built, result.BuiltFrom = built.Module, built.Commit
	controlPlaneImage := built.Image
	if o.DryRun {
		// Nothing was built, so there is no id to name. The carried builder's own is used only so
		// the remaining steps have something well-formed to describe; nothing is applied.
		controlPlaneImage = loaded.ID
	}

	// ---- 4. bundle ----------------------------------------------------------------------
	say("bundle — what this machine will be asked to be")
	rewritten, err := Rewrite(template, controlPlaneImage)
	if err != nil {
		return result, failed(StepBundle, err)
	}
	result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out
	result.Temporary = rewritten.TempName
	if rewritten.Renamed {
		say(fmt.Sprintf("  control plane     %s, renamed from %s",
			rewritten.TempName, rewritten.WasCalled))
		say("                    the permanent one is a module and takes the plain name; " +
			"this one is dropped at the end")
	} else {
		say("  control plane     " + rewritten.TempName + " — the template already named it that")
	}
	if rewritten.Changed {
		say(fmt.Sprintf("  its image         %s", rewritten.Now))
		say(fmt.Sprintf("  replacing         %s, named in %d place(s)",
			rewritten.Was, rewritten.Places))
	} else {
		say(fmt.Sprintf("  its image         %s — the template already named it, nothing rewritten",
			rewritten.Now))
	}
	if loaded.Predicted {
		say("  UNCONFIRMED       that id is the archive's and no runtime has been asked. A real " +
			"run reads it back.")
	}
	for _, kept := range rewritten.Kept {
		say("  left alone        " + kept)
	}
	if rewritten.BrokerAddress != "" {
		// Said every time, and never changed. Every enrolment token this mesh issues will tell a
		// joining node to dial this address, and a wrong one is silent until the second node fails
		// to come back. The installer does not know this machine's address and will not invent it.
		say("  nodes will dial   " + rewritten.BrokerAddress +
			" — check this is an address other machines can reach")
	}

	if o.DryRun {
		// Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it
		// if wrong while changing nothing. The bundle has been produced and re-parsed in memory,
		// which is everything that could be checked without touching the machine; what is left is
		// loading, applying and asking the result questions, and none of those can be answered by
		// not doing them.
		say(fmt.Sprintf("  would write       %s (%d resources)", o.Out, rewritten.Resources))
		if o.pivots() {
			// Named rather than attempted. Everything from step 6 on is a conversation with a
			// control plane that a dry run has not raised, so there is nothing to ask and nothing
			// honest to report about the answers.
			say("  would then        enrol " + o.Node + ", install the registry from " +
				o.Catalogue + ", push the control plane's image into it,")
			say("                    reinstall the control plane as a module, and drop " +
				rewritten.TempName)
		}
		result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " +
			"loaded or applied"
		if loaded.Predicted {
			result.Stopped += ". The image id in it is the archive's own and is not necessarily " +
				"the one this machine would hold — a runtime rewrites an image's configuration as " +
				"it loads, and the id is that configuration's digest"
		}
		say("\n" + result.Stopped)
		return result, nil
	}

	// **Nothing predicted is ever written down.** The line above is the only path on which
	// `loaded.ID` can be the archive's id, and it returns. Asserted here rather than left to the
	// reader, because what would follow is a bundle naming an image this machine does not hold —
	// and nothing serves an image named by the digest of its own configuration, so it would fail
	// inside a pull that cannot succeed, three steps from the cause.
	if loaded.Predicted {
		return result, failed(StepBundle, fmt.Errorf(
			"the builder's image id was never confirmed against this machine's runtime, and the "+
				"build was about to be run with it. This is a fault in the installer, not in the "+
				"machine"))
	}

	if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil {
		return result, failed(StepBundle, err)
	}
	result.BundleWrote = true
	say(fmt.Sprintf("  wrote             %s (%d resources) — read it, this is what is applied",
		o.Out, rewritten.Resources))

	// ---- 4. apply -----------------------------------------------------------------------
	say("apply — raising the substrate")
	report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say)
	result.Applied, result.Changed = len(report.Outcomes), report.Changed()
	if err != nil {
		return result, failed(StepApply, err)
	}
	if report.Changed() {
		say(fmt.Sprintf("  applied           %d resource(s)", len(report.Outcomes)))
	} else {
		say(fmt.Sprintf("  already matches   %d resource(s) checked, nothing moved",
			len(report.Outcomes)))
	}

	// ---- 5. verify ----------------------------------------------------------------------
	say("verify — the substrate is up, and the control plane replies")
	verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say)
	result.Running, result.Answered = verified.Running, verified.Answered
	if err != nil {
		return result, failed(StepVerify, err)
	}

	if !o.pivots() {
		// Stopped, and said plainly. What has been raised works and cannot be upgraded: its
		// control plane is named by an image id, which no registry serves, so nothing can ever
		// replace it with a newer one. That is the whole of what the pivot fixes, and it needs
		// manifests, and manifests come from a checkout somebody has to point this at.
		result.Stopped = "no --catalog was given, so this stopped at the substrate. " +
			"The control plane is named by the digest of its own configuration and no registry " +
			"serves it, so this mesh cannot yet upgrade itself. Run again with " +
			"--catalog <a checkout of the mesh's catalogue> to finish the pivot; every step " +
			"above will say it is already done"
		say("\nthis machine is a mesh of one node, with nothing joined to it yet.")
		say(result.Stopped)
		return result, nil
	}

	// ---- 6. enrol -------------------------------------------------------------------------
	//
	// From here on the mesh is being told things, and the way to tell it anything is to run its
	// own binary inside its own container. `temporary` is the substrate's control plane; the
	// module's is a different container with a different name and does not exist yet.
	temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout}

	say("enrol — this machine joins the mesh it is running")
	enrolled, err := Enrol(ctx, o, sys, temporary, say)
	result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined
	result.Agent = enrolled.Agent
	if err != nil {
		return result, failed(StepEnrol, err)
	}

	// ---- 7. registry ----------------------------------------------------------------------
	say("registry — somewhere for this mesh to keep its own images")
	registry, err := InstallRegistry(ctx, o, d, temporary, say)
	result.Registry, result.RegistryRunning = registry.Address, registry.Container
	result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered
	if err != nil {
		return result, failed(StepRegistry, err)
	}

	// ---- 8. publish -----------------------------------------------------------------------
	say("publish — the control plane's image gets its first manifest digest")
	// **The image this mesh built, not the one the installer carried.** The carried one is the
	// builder; publishing it here would put a builder in the registry under the control plane's
	// name and install it as the control plane — which is exactly what happened the first time
	// this ran, and presented as a control plane that started, printed a builder's usage, and
	// exited cleanly over and over.
	published, err := PublishControlPlane(ctx, o, d, controlPlaneImage, say)
	result.PublishedAs, result.PublishedAlready = published.Reference, published.Already
	if err != nil {
		return result, failed(StepPublish, err)
	}

	// ---- 9. control plane -----------------------------------------------------------------
	say("control plane — installed as an ordinary module, pinned to that digest")
	permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration,
		published.Reference, say)
	result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered
	result.StoresDelivered = permanent.Delivered
	if err != nil {
		return result, failed(StepControlPlane, err)
	}

	// ---- 11. retire -----------------------------------------------------------------------
	say("retire — the temporary control plane is dropped from the bundle")
	retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say)
	result.TemporaryRetired = retired.Gone || retired.Already
	result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed
	if err != nil {
		return result, failed(StepRetire, err)
	}

	// ---- 12. builder ----------------------------------------------------------------------
	//
	// **Last, and part of installing rather than after it.** What the steps above produce is a mesh
	// that runs and cannot make anything — every module in the catalogue names artifacts and
	// nothing has built them. The builder is carried in this installer, because it is what built
	// the control plane; putting it in the registry and installing it as a module is what turns a
	// mesh that runs into a mesh that can produce.
	say("builder — the mesh gets the thing that makes everything else")
	// The PERMANENT control plane, not the temporary one: by here the temporary is gone, and the
	// module this installs is assigned through the thing that will still be running afterwards.
	permanentControl := controlPlane{container: ControlPlaneModule, run: d.Run, timeout: o.Timeout}
	builder, err := InstallBuilder(ctx, o, d, permanentControl, loaded.ID, say)
	result.BuilderPublished = builder.Published.Reference
	result.BuilderInstalled = builder.Installed.Assigned || builder.Installed.Known
	result.BuilderAccount = builder.Account
	if err != nil {
		return result, failed(StepBuilder, err)
	}

	// ---- packages — the registry, before the base that resolves the SDK from it ----------
	say("packages — a registry answers, and the SDK is in it, before the base is built")
	if err := RaisePackageRegistry(ctx, o, d, permanentControl, say); err != nil {
		return result, failed(StepPackages, err)
	}

	// ---- sdk — published on a public base, so this needs no toolchain --------------------
	say("sdk — the shared library, published by version so the base can resolve it")
	if err := PublishTheSDK(ctx, o, permanentControl, say); err != nil {
		return result, failed(StepSDK, err)
	}

	// ---- 13. base -------------------------------------------------------------------------
	say("base — the shared toolchain and runtime everything with code stands on")
	if err := BuildBase(ctx, o, permanentControl, say); err != nil {
		return result, failed(StepBase, err)
	}

	// ---- 14. store ------------------------------------------------------------------------
	// A database PROVIDER. The substrate's store is the control plane's own memory and offers
	// nothing to anything; the first thing that wants a database is the catalogue, next.
	say("store — a database provider, which the substrate's own store is not")
	if err := InstallFromCatalogue(ctx, o, permanentControl, "postgres", say); err != nil {
		return result, failed(StepStore, err)
	}

	// ---- 15. catalogue --------------------------------------------------------------------
	say("catalogue — the module graph: what is held, what a change reaches, what to rebuild")
	if err := InstallFromCatalogue(ctx, o, permanentControl, "mesh-catalog", say); err != nil {
		return result, failed(StepCatalogue, err)
	}

	// ---- 16. network ----------------------------------------------------------------------
	say("network — the private network, and this machine's name on it")
	if err := PlaceOnTheNetwork(ctx, o, permanentControl, rewritten.BrokerAddress, say); err != nil {
		return result, failed(StepNetwork, err)
	}

	// ---- 17. filter -----------------------------------------------------------------------
	say("filter — required, so the question is which, not whether")
	if err := ChooseAndInstallFilter(ctx, o, permanentControl, say); err != nil {
		return result, failed(StepFilter, err)
	}

	// ---- 18. extras -----------------------------------------------------------------------
	say("extras — beyond the floor, if asked")
	if err := InstallExtras(ctx, o, permanentControl, say); err != nil {
		return result, failed(StepExtras, err)
	}

	say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " +
		"sits on its private network, and filters what modules declared.")
	say("what remains is somebody else's: adding nodes, and assigning what they should run.")

	return result, nil
}