mesh-host/internal/bootstrap/control.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	"github.com/novox/mesh-host/internal/declaration"
)

// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the
// environment.
//
// `MESH_STORE_<CONTEXT>` is what the control plane reads (mesh-controller's `internal/store`.Variable)
// and putting a password in a container's environment puts it in `docker inspect` for ever. So a
// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value
// into that file on the machine, and nothing but the process reads it.
const storeFileSuffix = "_FILE"

// storeVariablePrefix is the front of the same names.
const storeVariablePrefix = "MESH_STORE_"

// Permanent is what step 9 did.
type Permanent struct {
	Installed
	// Container is what the module calls its container, confirmed running.
	Container string
	// Image is what it is pinned to — the digest step 8's push produced.
	Image string
	// Delivered is every store connection accepted into it, by secret name.
	Delivered []string
	// Answered is what the permanent control plane said back.
	Answered string
}

// InstallControlPlane makes the control plane an ordinary module.
//
// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary
// control plane composes a declaration naming the registry-pinned image, publishes it, and this
// node's host creates the container. Nothing is asked to replace itself while running, which is
// what makes the whole thing expressible: the container being created is called `mesh-controller` and
// the one composing it is called `temp-mesh-controller`, so there are two of them and neither is in
// the other's way.
//
// **The store connections are the foundation's, made at genesis, and the mesh cannot invent them.**
// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are
// the credentials the foundation bundle created the databases with. Generating replacements would
// put thirty-two random bytes where a working connection string has to be, and the control plane
// would come up unable to open a single context. So they go in through `secret accept`, which is
// exactly the path for a value the mesh must carry and could not have invented — and they are read
// out of the bundle this installer produced rather than reconstructed, because the bundle is what
// created them and a second opinion about what a DSN should say is a second chance to be wrong.
//
// **The manifest is the one the build produced** — the manifest at the root of the control plane's
// own repository, its artifact resolved to the image built at step 3 (novox/hq ADR 0069). The
// installer used to read a second copy out of the catalogue and pin its placeholder; the copies
// drifted, and the first rebuild from source replaced the mesh's record with the repository's shape
// while every later push was refused on its behalf (novox/hq 04-ISSUES/072). There is one manifest
// now, and the only thing this step changes in it is the image's name: the id the machine built it
// under becomes the reference the registry assigned at step 8.
func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane,
	foundation *declaration.Declaration, built Built, image string, say func(string)) (Permanent, error) {

	out := Permanent{Image: image}

	if len(built.Manifest) == 0 {
		return out, fmt.Errorf(
			"the control plane was not built, so there is no manifest to register it with. " +
				"This is the manifest that makes the control plane an ordinary module. Without it " +
				"the machine keeps the temporary control plane the foundation raised, which works " +
				"and cannot be upgraded — so the install stops here rather than pretending to " +
				"have pivoted")
	}

	pinned, places, err := pinImage(built.Manifest, built.Image, image, ControlPlaneModule)
	if err != nil {
		return out, err
	}
	say(fmt.Sprintf("  pinned            %s → %s, in %d place(s)", shortRef(built.Image), image, places))

	container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned))
	if err != nil {
		return out, err
	}
	out.Container = container
	if container == "" {
		return out, fmt.Errorf(
			"the %s module's container has no name, so nothing can be verified afterwards",
			ControlPlaneModule)
	}

	installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say)
	out.Installed = installed
	if err != nil {
		return out, err
	}

	// The connections, before the push that would otherwise deliver random bytes for them.
	delivered, err := deliverStores(ctx, o, control, pinned, foundation, say)
	out.Delivered = delivered
	if err != nil {
		return out, err
	}

	if out.Pushed, err = pushNode(ctx, o, control, say); err != nil {
		return out, err
	}

	if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil {
		return out, err
	}
	// And it answers, which is the same question step 5 asked of the temporary one and for the
	// same reason: `status` opens all three stores, so a reply proves the sealed connections it
	// was given are the ones the foundation made. Asked of the NEW container — this is the only
	// moment in the program where two control planes are running, and asking the wrong one would
	// report the temporary one's health as the permanent one's.
	answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say)
	if err != nil {
		return out, err
	}
	out.Answered = answered
	return out, nil
}

// pinImage replaces the image the build named with the reference the registry assigned.
//
// **Textual, and every place it appears.** A manifest may name its image in more than one resource
// — a migrate step beside the server, a runtime beside the application — and the same reasoning
// as the bundle rewrite applies: replacing one and not the others leaves something pointing at an
// image nothing serves, and it fails half way through an apply rather than here.
//
// The built image is named by the digest of its own configuration, `sha256:…` with no registry in
// front, which only the machine that built it can resolve. The whole JSON string moves to the
// registry's `<registry>/<repository>@sha256:…`, so every machine the module is later pushed to
// pulls it from the mesh's own store.
//
// It refuses a manifest that does not name the built image. That is not pedantry: a manifest
// naming some other image is one that describes some other build, and quietly registering it would
// install a control plane that is not the image this machine just published — which is the one
// thing this step exists to guarantee.
func pinImage(manifest []byte, built, reference, module string) ([]byte, int, error) {
	from := []byte(`"` + built + `"`)
	places := bytes.Count(manifest, from)
	if places == 0 {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest does not name the image this machine built (%s), so there "+
				"is nothing to pin to the image it just published.\n"+
				"A manifest naming some other image describes some other build. Registering it "+
				"would install a module that is not the one this installer built and pushed",
			module, built)
	}
	pinned := bytes.ReplaceAll(manifest, from, []byte(`"`+reference+`"`))

	// Read back. A substitution on text can catch more than it was aimed at, and the manifest is
	// about to be handed to the mesh as the description of what it runs.
	var checked map[string]any
	if err := json.Unmarshal(pinned, &checked); err != nil {
		return nil, 0, fmt.Errorf(
			"pinning the %s module's image broke its manifest: %w", module, err)
	}
	if bytes.Contains(pinned, from) {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest still names the built image after pinning", module)
	}
	return pinned, places, nil
}

// controlPlaneResourceIn is the id of the resource that runs the control plane.
//
// The manifest is the control plane's own and the installer does not get to name its resources.
// What it can do is find the one container whose image is the one just pinned — and when a manifest
// declares exactly one container, that is the answer without any searching at all.
func controlPlaneResourceIn(manifest []byte) string {
	var m struct {
		Resources []struct {
			ID   string `json:"id"`
			Type string `json:"type"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return ""
	}
	var containers []string
	for _, r := range m.Resources {
		if r.Type == "container" {
			containers = append(containers, r.ID)
		}
	}
	if len(containers) == 1 {
		return containers[0]
	}
	// More than one, so the name has to be guessed at rather than derived — and the catalogue's
	// own convention for the resource that IS the module is `container`, with anything else beside
	// it named for what it does.
	for _, id := range containers {
		if id == "container" || id == ControlPlaneModule || id == "control-plane" {
			return id
		}
	}
	return ""
}

// deliverStores carries the foundation's own database connections into the module.
//
// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls
// these secrets is what is delivered. The installer does not guess that the secret holding the
// inventory connection is called `inventory`; it follows the manifest from the variable to the
// secret, and a manifest whose two ends do not meet is refused rather than half-delivered.
//
// **What is delivered is what the foundation already has, and only that.** The mesh generates an
// own-secret nobody supplied, which is right for something coming into existence and wrong for
// something that already exists. So every variable the module fills from a secret is looked up in
// the foundation's control plane: what it names is accepted, what it does not is left for the mesh
// to make. A store connection missing from the foundation is the one exception and is an error —
// a control plane that cannot open a context is not a control plane.
func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte,
	foundation *declaration.Declaration, say func(string)) ([]string, error) {

	wanted, err := secretsByVariableIn(manifest)
	if err != nil {
		return nil, err
	}
	if !anyStoreIn(wanted) {
		return nil, fmt.Errorf(
			"the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+
				"each context through its own credential (novox/hq ADR 0008), so a manifest naming "+
				"none describes a control plane that can open nothing.\n"+
				"The shape this installer delivers into is a file per context, named by an "+
				"own-secret, with %s<CONTEXT>%s pointing at it — directly, or at where that file is "+
				"mounted inside the container",
			ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix)
	}

	temporary, err := controlPlaneIn(foundation)
	if err != nil {
		return nil, err
	}

	var delivered []string
	for _, variable := range sortedKeys(wanted) {
		secret := wanted[variable]
		value := strings.TrimSpace(temporary.Env[variable])
		if value == "" {
			if strings.HasPrefix(variable, storeVariablePrefix) {
				return delivered, fmt.Errorf(
					"the %s module wants %s and the bundle this installer produced does not name "+
						"one.\n"+
						"That connection is the foundation's, created at genesis — the mesh cannot "+
						"invent it and the installer will not guess at one",
					ControlPlaneModule, variable)
			}
			// Not something the foundation made. The mesh generates its own, which is exactly what
			// an own-secret is for; said so that nothing about the delivery is silent.
			say("  the mesh will make " + secret + " — the foundation names no " + variable)
			continue
		}

		// Into the container as a file, because `secret accept` reads a file or a prompt and the
		// installer has neither a terminal to be prompted at nor a way to write to a command's
		// standard input through the runner every applier in this repository shares.
		at := "/accepting-" + secret
		if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
			return delivered, err
		}
		if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret,
			"--from", at); err != nil {
			return delivered, err
		}
		delivered = append(delivered, secret)
		say("  accepted          " + secret + " — " + variable + ", as the foundation made it")
	}
	return delivered, nil
}

// secretsByVariableIn maps each environment variable the module fills from a secret to that
// secret's name.
//
// Two shapes, because the catalogue uses both:
//
//   - `MESH_STORE_<CONTEXT>_FILE` in the container's environment, naming a path the process reads.
//     The path may be the own-secret's own path, or — more usually — where that file is mounted
//     inside the container, in which case the volumes say which is which. Following the mount is
//     not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at
//     `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and
//     refuse a correct manifest.
//   - `VAR=${secret:name}` inside a file resource the container reads its environment from, which
//     is how a value that is not a path gets in at all.
//
// A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and
// the process would find an empty file where a credential has to be, which presents as a container
// that will not start, a long way from the cause.
func secretsByVariableIn(manifest []byte) (map[string]string, error) {
	var m struct {
		OwnSecrets map[string]string `json:"own-secrets"`
		Resources  []struct {
			Type    string            `json:"type"`
			Path    string            `json:"path"`
			Content string            `json:"content"`
			Env     map[string]string `json:"env"`
			Volumes []string          `json:"volumes"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
	}

	secretAt := map[string]string{}
	for name, path := range m.OwnSecrets {
		secretAt[path] = name
	}

	wanted := map[string]string{}
	for _, r := range m.Resources {
		switch r.Type {
		case "file":
			for variable, secret := range secretsInContent(r.Content) {
				wanted[variable] = secret
			}
		case "container":
			inside := mountedFrom(r.Volumes)
			for key, path := range r.Env {
				// Any `MESH_…_FILE` naming an own-secret's file, not only the store's: the broker
				// settings took the same shape once a secret stopped travelling in an env-file
				// (novox/hq ADR 0086, issue 041).
				if !strings.HasPrefix(key, "MESH_") || !strings.HasSuffix(key, storeFileSuffix) {
					continue
				}
				on := path
				if from, mounted := inside[path]; mounted {
					on = from
				}
				secret, named := secretAt[on]
				if !named {
					return nil, fmt.Errorf(
						"the %s module's container reads %s from %s, and no own-secret of that "+
							"module writes that file.\n"+
							"So the mesh would seal nothing there and the control plane would find "+
							"an empty file where a connection string has to be. The manifest has to "+
							"name the two ends the same, directly or through a mount",
						ControlPlaneModule, key, path)
				}
				wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret
			}
		}
	}
	return wanted, nil
}

// mountedFrom is where each path inside a container comes from outside it.
func mountedFrom(volumes []string) map[string]string {
	inside := map[string]string{}
	for _, volume := range volumes {
		parts := strings.Split(volume, ":")
		if len(parts) < 2 {
			continue
		}
		inside[parts[1]] = parts[0]
	}
	return inside
}

// secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment
// from.
func secretsInContent(content string) map[string]string {
	found := map[string]string{}
	for _, line := range strings.Split(content, "\n") {
		variable, value, is := strings.Cut(strings.TrimSpace(line), "=")
		if !is {
			continue
		}
		const opens = "${secret:"
		if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") {
			continue
		}
		found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}")
	}
	return found
}

// anyStoreIn reports whether any of these variables is a context's connection.
func anyStoreIn(wanted map[string]string) bool {
	for variable := range wanted {
		if strings.HasPrefix(variable, storeVariablePrefix) {
			return true
		}
	}
	return false
}

func sortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}