mesh-host/internal/bootstrap/verify.go

package bootstrap

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// controlPlaneBinary is where the control plane's program lives in its own image.
//
// A path rather than a shell command, because the image is `FROM scratch` and holds one static
// binary and nothing else — no shell to invoke, nothing to interpret a command line
// (mesh-controller's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer
// carries, which is why the path can be written down here.
const controlPlaneBinary = "/mesh-controller"

// answerEvery is how often the control plane is asked again while it is starting.
var answerEvery = 2 * time.Second

// Verified is what the foundation was found to be.
type Verified struct {
	// Running is every long-running container the bundle declares, confirmed up.
	Running []string
	// Answered is the control plane's own first words back, so the report shows the reply rather
	// than asserting there was one.
	Answered string
}

// Verify proves the foundation is up and the control plane replies.
//
// **A container that is up is not a control plane that replies**, and this project has paid for
// that distinction more than once: a runtime reports a container running from the moment the
// process starts, which is before it has opened a database, before it has read its configuration,
// and before it has failed to. So the containers are checked, and then the program inside one of
// them is asked a question and has to answer it.
//
// The question is `status`, and it is chosen rather than convenient: answering it means the
// control plane opened all three of its stores from the environment the bundle gave it. A reply
// therefore proves the image runs, that `network: host` really does reach the store on this
// machine, and that the contexts' schemas migrated — the three things the steps before this were
// for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server.
//
// It waits. A control plane that is not answering yet and a control plane that will never answer
// look identical for the first few seconds, and refusing on the first attempt would make a correct
// bootstrap fail for being observed too early.
func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration,
	say func(string)) (Verified, error) {

	var out Verified

	control, err := controlPlaneIn(d)
	if err != nil {
		return out, err
	}

	for _, container := range longRunning(d) {
		state, err := containerRunning(ctx, run, probe, container)
		if err != nil {
			return out, err
		}
		if !state.running {
			return out, fmt.Errorf(
				"the container %q is %s, not running.\n"+
					"The apply reported success, so it was created — what it did afterwards is "+
					"in `docker logs %s`", container, state.status, container)
		}
		out.Running = append(out.Running, container)
		say("  running           " + container)
	}

	answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say)
	if err != nil {
		return out, err
	}
	out.Answered = answer
	return out, nil
}

func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration,
	container string, say func(string)) (string, error) {

	deadline := time.Now().Add(wait)
	var last error
	for {
		asking, cancel := context.WithTimeout(ctx, probe)
		out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status")
		cancel()

		answer := strings.TrimSpace(firstLineOf(out))
		switch {
		case err != nil:
			last = err
		case answer == "":
			// Exit zero and nothing said. Treated as no answer rather than as success: a program
			// that returns silence is not one that has been asked anything.
			last = fmt.Errorf("it exited without saying anything")
		default:
			say("  replies           " + container + ": " + answer)
			return answer, nil
		}

		if time.Now().After(deadline) {
			break
		}
		select {
		case <-ctx.Done():
			return "", ctx.Err()
		case <-time.After(answerEvery):
		}
	}
	return "", fmt.Errorf(
		"the container %q is running and the control plane in it does not answer, after waiting "+
			"%s: %v\n"+
			"Running is not replying. `status` opens this mesh's three stores, so what failed is "+
			"most likely the store or the schemas rather than the control plane itself — "+
			"`docker logs %s` says which", container, wait, last, container)
}

type containerState struct {
	running bool
	status  string
}

func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) {
	asking, cancel := context.WithTimeout(ctx, probe)
	defer cancel()

	// Both facts in one answer, so a container that is not running is reported with what it IS
	// rather than with the absence of what it should be. `container inspect`, not the bare form:
	// a same-named network or volume would otherwise answer in the container's place.
	out, err := run(asking, "docker", "container", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name)
	if err != nil {
		return containerState{}, fmt.Errorf(
			"the container %q is not there at all, and the apply reported it applied: %w", name, err)
	}
	running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ")
	if status == "" {
		status = "in a state the runtime did not name"
	}
	return containerState{running: running == "true", status: status}, nil
}

// longRunning is every container the bundle expects to still be there afterwards.
//
// A run-once step has exited by design and a scheduled step has deliberately never been started
// (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the
// foundation to be something other than what it declared.
func longRunning(d *declaration.Declaration) []string {
	var names []string
	for _, r := range d.Resources {
		container, ok := r.(*declaration.Container)
		if !ok || container.RunOnce || container.Schedule != "" {
			continue
		}
		names = append(names, container.Name)
	}
	return names
}

func firstLineOf(s string) string {
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		return s[:i]
	}
	return s
}