mesh-host/internal/bootstrap/control.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	"github.com/novox/mesh-host/internal/declaration"
)

// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the
// environment.
//
// `MESH_STORE_<CONTEXT>` is what the control plane reads (mesh-control's `internal/store`.Variable)
// and putting a password in a container's environment puts it in `docker inspect` for ever. So a
// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value
// into that file on the machine, and nothing but the process reads it.
const storeFileSuffix = "_FILE"

// storeVariablePrefix is the front of the same names.
const storeVariablePrefix = "MESH_STORE_"

// Permanent is what step 9 did.
type Permanent struct {
	Installed
	// Container is what the module calls its container, confirmed running.
	Container string
	// Image is what it is pinned to — the digest step 8's push produced.
	Image string
	// Delivered is every store connection accepted into it, by secret name.
	Delivered []string
	// Answered is what the permanent control plane said back.
	Answered string
}

// InstallControlPlane makes the control plane an ordinary module.
//
// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary
// control plane composes a declaration naming the registry-pinned image, publishes it, and this
// node's host creates the container. Nothing is asked to replace itself while running, which is
// what makes the whole thing expressible: the container being created is called `mesh-control` and
// the one composing it is called `temp-mesh-control`, so there are two of them and neither is in
// the other's way.
//
// **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.**
// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are
// the credentials the substrate bundle created the databases with. Generating replacements would
// put thirty-two random bytes where a working connection string has to be, and the control plane
// would come up unable to open a single context. So they go in through `secret accept`, which is
// exactly the path for a value the mesh must carry and could not have invented — and they are read
// out of the bundle this installer produced rather than reconstructed, because the bundle is what
// created them and a second opinion about what a DSN should say is a second chance to be wrong.
func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane,
	substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) {

	out := Permanent{Image: image}

	manifest, err := readManifest(o.Catalogue, ControlPlaneModule)
	if err != nil {
		return out, fmt.Errorf(
			"%w\n"+
				"This is the manifest that makes the control plane an ordinary module. Without it "+
				"the machine keeps the temporary control plane the substrate raised, which works "+
				"and cannot be upgraded — so the install stops here rather than pretending to "+
				"have pivoted", err)
	}

	pinned, places, err := pinImage(manifest, image)
	if err != nil {
		return out, err
	}
	say(fmt.Sprintf("  pinned            %s, in %d place(s)", image, places))

	container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned))
	if err != nil {
		return out, err
	}
	out.Container = container
	if container == "" {
		return out, fmt.Errorf(
			"the %s module's container has no name, so nothing can be verified afterwards",
			ControlPlaneModule)
	}

	installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say)
	out.Installed = installed
	if err != nil {
		return out, err
	}

	// The connections, before the push that would otherwise deliver random bytes for them.
	delivered, err := deliverStores(ctx, o, control, pinned, substrate, say)
	out.Delivered = delivered
	if err != nil {
		return out, err
	}

	if out.Pushed, err = pushNode(ctx, o, control, say); err != nil {
		return out, err
	}

	if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil {
		return out, err
	}
	// And it answers, which is the same question step 5 asked of the temporary one and for the
	// same reason: `status` opens all three stores, so a reply proves the sealed connections it
	// was given are the ones the substrate made. Asked of the NEW container — this is the only
	// moment in the program where two control planes are running, and asking the wrong one would
	// report the temporary one's health as the permanent one's.
	answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say)
	if err != nil {
		return out, err
	}
	out.Answered = answered
	return out, nil
}

// pinImage replaces the catalogue's placeholder digest with what the registry assigned.
//
// **Textual, and every place it appears.** A manifest may name its image in more than one resource
// — the catalogue's converted modules routinely carry a runtime container beside the application's
// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves
// something pointing at an image nothing serves, and it fails half way through an apply rather
// than here.
//
// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already
// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a
// control plane that is not the image this machine just published — which is the one thing this
// step exists to guarantee.
func pinImage(manifest []byte, reference string) ([]byte, int, error) {
	places := bytes.Count(manifest, []byte(placeholderDigest))
	if places == 0 {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+
				"pin to the image this machine just published.\n"+
				"A manifest already naming a digest was pinned by somebody else, to some other "+
				"build. Registering it would install a control plane that is not the one this "+
				"installer carried and pushed", ControlPlaneModule, placeholderDigest)
	}
	// The reference the registry gave back is `<registry>/<repository>@sha256:…`, and what the
	// manifest holds is `<something>@sha256:0…0`. Replacing only the digest would leave the
	// manifest's own repository name in front of it — which may be `mesh-control` with no
	// registry, and a runtime would then pull it from the internet. The whole reference moves.
	var out bytes.Buffer
	rest := manifest
	for {
		at := bytes.Index(rest, []byte(placeholderDigest))
		if at < 0 {
			out.Write(rest)
			break
		}
		// Back up over the repository this digest belongs to, which runs to the opening quote.
		start := bytes.LastIndexByte(rest[:at], '"')
		if start < 0 {
			return nil, 0, fmt.Errorf(
				"the %s module's manifest has a placeholder digest that is not inside a JSON "+
					"string, so the installer cannot tell what image it belongs to", ControlPlaneModule)
		}
		out.Write(rest[:start+1])
		out.WriteString(reference)
		rest = rest[at+len(placeholderDigest):]
	}
	pinned := out.Bytes()

	// Read back. A substitution on text can catch more than it was aimed at, and the manifest is
	// about to be handed to the mesh as the description of what it runs.
	var checked map[string]any
	if err := json.Unmarshal(pinned, &checked); err != nil {
		return nil, 0, fmt.Errorf(
			"pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err)
	}
	if bytes.Contains(pinned, []byte(placeholderDigest)) {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest still carries a placeholder digest after pinning",
			ControlPlaneModule)
	}
	return pinned, places, nil
}

// controlPlaneResourceIn is the id of the resource that runs the control plane.
//
// The manifest is written by the catalogue and the installer does not get to name its resources.
// What it can do is find the one container whose image is the one just pinned — and when a manifest
// declares exactly one container, that is the answer without any searching at all.
func controlPlaneResourceIn(manifest []byte) string {
	var m struct {
		Resources []struct {
			ID   string `json:"id"`
			Type string `json:"type"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return ""
	}
	var containers []string
	for _, r := range m.Resources {
		if r.Type == "container" {
			containers = append(containers, r.ID)
		}
	}
	if len(containers) == 1 {
		return containers[0]
	}
	// More than one, so the name has to be guessed at rather than derived — and the catalogue's
	// own convention for the resource that IS the module is `container`, with anything else beside
	// it named for what it does.
	for _, id := range containers {
		if id == "container" || id == ControlPlaneModule || id == "control-plane" {
			return id
		}
	}
	return ""
}

// deliverStores carries the substrate's own database connections into the module.
//
// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls
// these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a
// path, and the module's own-secret that writes that path is the secret to accept the connection
// as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the
// secret is called `inventory`, would seal a connection string under a name nothing reads and
// leave the mesh to invent random bytes for the one that is.
func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte,
	substrate *declaration.Declaration, say func(string)) ([]string, error) {

	wanted, err := storeSecretsIn(manifest)
	if err != nil {
		return nil, err
	}
	if len(wanted) == 0 {
		return nil, fmt.Errorf(
			"the %s module's manifest asks for no store connections. A control plane reaches each "+
				"context through its own credential (novox/hq ADR 0008), so a manifest naming none "+
				"describes a control plane that can open nothing.\n"+
				"The shape this installer delivers into is a file per context, named by an "+
				"own-secret, with %s<CONTEXT>%s in the container's environment pointing at it",
			ControlPlaneModule, storeVariablePrefix, storeFileSuffix)
	}

	temporary, err := controlPlaneIn(substrate)
	if err != nil {
		return nil, err
	}

	var delivered []string
	for _, context := range sortedKeys(wanted) {
		secret := wanted[context]
		connection := temporary.Env[storeVariablePrefix+context]
		if strings.TrimSpace(connection) == "" {
			return delivered, fmt.Errorf(
				"the %s module wants the %s store's connection and the bundle this installer "+
					"produced does not name one: its control plane has no %s.\n"+
					"These connections are the substrate's, created at genesis — the mesh cannot "+
					"invent them and the installer will not guess at one",
				ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context)
		}

		// Into the container as a file, because `secret accept` reads a file or a prompt and the
		// installer has neither a terminal to be prompted at nor a way to write to a command's
		// standard input through the runner every applier in this repository shares.
		at := "/accepting-" + strings.ToLower(context)
		if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context),
			[]byte(connection), at); err != nil {
			return delivered, err
		}
		if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret,
			"--from", at); err != nil {
			return delivered, err
		}
		delivered = append(delivered, secret)
		say("  accepted          " + secret + " — the " + strings.ToLower(context) +
			" store, as the substrate made it")
	}
	return delivered, nil
}

// storeSecretsIn pairs each context with the secret its connection must be accepted as.
//
// Read out of the manifest twice over: the container's environment says which contexts are wanted
// and what file each expects, and the module's own-secrets say which secret writes which file. A
// pair that does not meet is refused rather than half-delivered.
func storeSecretsIn(manifest []byte) (map[string]string, error) {
	var m struct {
		OwnSecrets map[string]string `json:"own-secrets"`
		Resources  []struct {
			Type string            `json:"type"`
			Env  map[string]string `json:"env"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
	}

	byPath := map[string]string{}
	for name, path := range m.OwnSecrets {
		byPath[path] = name
	}

	wanted := map[string]string{}
	for _, r := range m.Resources {
		if r.Type != "container" {
			continue
		}
		for key, path := range r.Env {
			if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) {
				continue
			}
			context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix)
			secret, ok := byPath[path]
			if !ok {
				return nil, fmt.Errorf(
					"the %s module's container reads the %s store's connection from %s, and no "+
						"own-secret of that module writes that file.\n"+
						"So the mesh would seal nothing there and the control plane would find an "+
						"empty file where a connection string has to be. The manifest has to name "+
						"the two ends the same",
					ControlPlaneModule, strings.ToLower(context), path)
			}
			wanted[context] = secret
		}
	}
	return wanted, nil
}

func sortedKeys(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}