mesh-host/internal/bootstrap/module.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// placeholderDigest is what the catalogue writes where a built image's digest will go.
//
// Sixty-four zeros. A manifest in the catalogue names an image the mesh builds and pushes, and
// until that has happened there is no digest to name — so the convention is a digest that is
// obviously not one, replaced by the pipeline when it publishes. The installer meets it twice: it
// must NOT be there in the registry's manifest, whose image is upstream and never built
// (novox/hq 04-ISSUES/029), and it MUST be there in the control plane's, which is the image
// step 8 has just pushed.
const placeholderDigest = "sha256:" + "0000000000000000000000000000000000000000000000000000000000000000"

// catalogueDir is where a mesh-catalog checkout keeps its manifests.
const catalogueDir = "modules"

// manifestFile is the path a module's manifest is read from, given a catalogue checkout.
func manifestFile(catalogue, module string) string {
	return filepath.Join(catalogue, catalogueDir, module, "module.json")
}

// readManifest reads one module's manifest out of a mesh-catalog checkout.
//
// **From a checkout rather than from anything the mesh serves**, and that is the ordering the whole
// pivot exists to respect: at this point the mesh has no build machine, no forge and — until step 7
// finishes — no registry. What a manifest is, is a file; the installer is handed the directory it
// is in and reads it.
func readManifest(catalogue, module string) ([]byte, error) {
	path := manifestFile(catalogue, module)
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf(
			"the %s module's manifest could not be read: %w\n"+
				"--catalog is a checkout of the mesh's catalogue repository, and this is read from "+
				"%s inside it", module, err, filepath.Join(catalogueDir, module, "module.json"))
	}
	return raw, nil
}

// Installed is what installing one module did.
type Installed struct {
	// Module is its name.
	Module string
	// Known is true when the mesh already had this module in its catalogue. The manifest is
	// registered either way — a re-run with a changed manifest must land — so this reports what
	// was found rather than what was skipped.
	Known bool
	// Assigned is true when this node was given the module during this run.
	Assigned bool
	// Pushed is what the mesh said when it sent this node its declaration.
	Pushed string
}

// installModule registers a manifest, gives it to this node, and sends it.
//
// The three verbs a person types, in the order they type them, through the same commands. There is
// no installer-only path into the mesh: everything here is `module add`, `assign` and `push`, so
// what the installer does on a bare machine and what an operator does on a running mesh are the
// same act (novox/hq ADR 0035, one refusal per act however it is asked for).
func installModule(ctx context.Context, o Options, control controlPlane, module string,
	manifest []byte, say func(string)) (Installed, error) {

	out, err := registerAndAssign(ctx, o, control, module, manifest, say)
	if err != nil {
		return out, err
	}
	out.Pushed, err = pushNode(ctx, o, control, say)
	return out, err
}

// registerAndAssign is the half of installing that happens before anything is sent.
//
// Split out because one module needs something in between: the control plane's own store
// connections have to be accepted before its declaration is composed, or the mesh would seal
// thirty-two random bytes into the file it expects a connection string in and the container would
// come up unable to open anything (mesh-controller's `secret accept`, and what it exists for).
//
// **`module add` is run every time and is not skipped when the module is already known.** It is an
// upsert on the manifest, and the manifest is exactly what changes between runs — step 9 registers
// the control plane with a digest that did not exist the first time. Skipping it because the name
// was already in the catalogue would silently pin the mesh to the previous image.
func registerAndAssign(ctx context.Context, o Options, control controlPlane, module string,
	manifest []byte, say func(string)) (Installed, error) {

	out := Installed{Module: module}

	known, err := control.tell(ctx, "module", "list")
	if err != nil {
		return out, err
	}
	out.Known = mentions(known, module)

	remote := "/" + module + "-module.json"
	if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
		return out, err
	}
	if _, err := control.tell(ctx, "module", "add", remote); err != nil {
		return out, err
	}
	if out.Known {
		say("  registered        " + module + " — the mesh already knew it; the manifest is now this one")
	} else {
		say("  registered        " + module)
	}

	assigned, err := control.tell(ctx, "assign", o.Node, module)
	if err != nil {
		return out, err
	}
	out.Assigned = true
	say("  assigned          " + module + " to " + o.Node)
	if refusal := strings.TrimSpace(assigned); strings.Contains(refusal, "but ") {
		// `assign` records what a person meant and says at once when the machine cannot host it.
		// Repeated rather than swallowed: the push below will apply everything else and this is
		// the only place the reason appears.
		say(indent(refusal))
	}
	if err := prepareModule(ctx, o, control, module, say); err != nil {
		return out, err
	}

	return out, nil
}

// pushNode sends this node everything it should be.
//
// Given the long wait rather than the probe timeout: a push composes every declaration this node
// should hold, seals every secret in them and publishes them, and on a first node that is the
// slowest thing the mesh does.
func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
	return pushing(ctx, o, control, false, say)
}

// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was
// sent (`push <node> --wait`, novox/hq ADR 0010): for a push whose content the very next step reads
// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a
// moment later (issue 615).
func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
	return pushing(ctx, o, control, true, say)
}

func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) {
	args := []string{"push", o.Node}
	patience := o.Wait
	if applied {
		args = append(args, "--wait", o.Wait.String())
		// The controller's own wait, and time to compose and send before it starts.
		patience = 2 * o.Wait
	}
	said, err := control.within(patience).tell(ctx, args...)
	if err != nil {
		return "", fmt.Errorf(
			"%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+
				"not been sent it. Nothing is half-applied — a push the mesh refused sent nothing "+
				"at all. Fix what it named and run this installer again: it will find the module "+
				"already registered and try the push again", err)
	}
	if applied {
		say("  pushed            " + o.Node + ", and it applied it")
	} else {
		say("  pushed            " + o.Node)
	}
	// **Every push may carry a new bus user, and until the adoption the bus hears of it only from here**
	// (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's
	// records and sealed to this node with the push; the bus reads its user list from a file, and until
	// the broker step adopts the bus as the nats module nothing but this installer writes that file. The
	// list was placed only when the node enrolled, so the build agent's account (step 13) never reached
	// the bus and the agent was refused ("Authorization Violation") until the sdk step gave up.
	if err := placeUntilAdopted(ctx, control, say); err != nil {
		return "", err
	}
	return strings.TrimSpace(said), nil
}

// pinPlaceholder replaces the catalogue's placeholder digest with what the registry assigned — the
// builder's manifest, which the catalogue still holds (the control plane's comes out of its own
// build, see pinImage).
//
// **Textual, and every place it appears.** A manifest may name its image in more than one resource
// — the catalogue's converted modules routinely carry a runtime container beside the application's
// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves
// something pointing at an image nothing serves, and it fails half way through an apply rather
// than here.
//
// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already
// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a
// control plane that is not the image this machine just published — which is the one thing this
// step exists to guarantee.
func pinPlaceholder(manifest []byte, reference, module string) ([]byte, int, error) {
	places := bytes.Count(manifest, []byte(placeholderDigest))
	if places == 0 {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+
				"pin to the image this machine just published.\n"+
				"A manifest already naming a digest was pinned by somebody else, to some other "+
				"build. Registering it would install a module that is not the one this "+
				"installer carried and pushed", module, placeholderDigest)
	}
	// The reference the registry gave back is `<registry>/<repository>@sha256:…`, and what the
	// manifest holds is `<something>@sha256:0…0`. Replacing only the digest would leave the
	// manifest's own repository name in front of it — which may be `mesh-controller` with no
	// registry, and a runtime would then pull it from the internet. The whole reference moves.
	var out bytes.Buffer
	rest := manifest
	for {
		at := bytes.Index(rest, []byte(placeholderDigest))
		if at < 0 {
			out.Write(rest)
			break
		}
		// Back up over the repository this digest belongs to, which runs to the opening quote.
		start := bytes.LastIndexByte(rest[:at], '"')
		if start < 0 {
			return nil, 0, fmt.Errorf(
				"the %s module's manifest has a placeholder digest that is not inside a JSON "+
					"string, so the installer cannot tell what image it belongs to", module)
		}
		out.Write(rest[:start+1])
		out.WriteString(reference)
		rest = rest[at+len(placeholderDigest):]
	}
	pinned := out.Bytes()

	// Read back. A substitution on text can catch more than it was aimed at, and the manifest is
	// about to be handed to the mesh as the description of what it runs.
	var checked map[string]any
	if err := json.Unmarshal(pinned, &checked); err != nil {
		return nil, 0, fmt.Errorf(
			"pinning the %s module's image broke its manifest: %w", module, err)
	}
	if bytes.Contains(pinned, []byte(placeholderDigest)) {
		return nil, 0, fmt.Errorf(
			"the %s module's manifest still carries a placeholder digest after pinning",
			module)
	}
	return pinned, places, nil
}

// prepareModule is what genesis tells the controller about a module on this node once it is
// assigned and before it is pushed: the ports this node gave it, and on an adopted node that the
// module is taken (novox/hq ADR 0100).
func prepareModule(ctx context.Context, o Options, control controlPlane, module string,
	say func(string)) error {
	if err := setFoundationSettings(ctx, o, control, module, say); err != nil {
		return err
	}
	return takeIfAdopted(ctx, o, control, module, say)
}