Register R92 as a bed: genesis on a runtime that trusts no registry (hq issue 92)

The lab's base image trusts the documentation ranges as plain-HTTP registries, so no genesis bed could
see the push to an untrusted registry that issue 92 is about (reviews 73 and 76 on mesh-host #91). The
bed takes that trust off the anchor before the installer runs and asks the runtime it trusts loopback
only, then runs genesis with --registry 192.0.2.10:5000 and holds that the control plane reaches the
registry, trusted by the mesh's name too.

The prover gains a bed kind: it builds the node-engine and its installer at each commit, the carried
builder from the pinned controller, and raises the bed; a bed that never reached the step it replays
skips, and a skip is an error, never the fault reproduced.

Not proved yet: genesis stops at step 6 on both sides, because the temporary controller has no
MESH_BUS_NATS and the bus template is refused by the installer (issue 146).

replays/cmd/prove/main.go

@@ -7,7 +7,9 @@
 // The clones are the core repositories, side by side as the lab expects them. A replay in its repository
 // is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
 // the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
-// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's.
+// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
+// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
+// through to the controller's.
 package main
 
 import (
@@ -20,6 +22,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -63,7 +67,12 @@ func main() {
 
 // run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
 func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	limit := 15 * time.Minute
+	if r.Kind == replays.Bed {
+		// Two machines raised, a foundation pulled and a control plane built inside one of them.
+		limit = 90 * time.Minute
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), limit)
 	defer cancel()
 	repo := filepath.Join(repos, r.Repository)
 	tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
@@ -77,6 +86,8 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
 	defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
 
 	switch r.Kind {
+	case replays.Bed:
+		return aBed(ctx, r, repos, tree, fixed)
 	case replays.InRepository, replays.Gate, replays.Liveness:
 		home := filepath.Join(repos, r.Repository)
 		if r.Home != "" {
@@ -155,6 +166,136 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
 	return "error", "no such kind"
 }
 
+// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
+// installer built from it, the installer carrying a builder built from the controller the replay pins, and
+// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
+func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
+	beside := map[string]string{}
+	for _, also := range r.With {
+		ref := also.Fix
+		if !fixed {
+			ref = also.Fix + "^1"
+			if also.Before != "" {
+				ref = also.Before
+			}
+		}
+		other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
+		if err != nil {
+			return "error", err.Error()
+		}
+		defer os.RemoveAll(other)
+		clone := filepath.Join(repos, also.Repository)
+		if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
+			ref).CombinedOutput(); err != nil {
+			return "error", string(out)
+		}
+		defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
+		beside[also.Repository] = other
+	}
+	controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
+	if controller == "" || catalogue == "" {
+		return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
+			"not both"
+	}
+	builder, err := builderImage(ctx, controller)
+	if err != nil {
+		return "error", err.Error()
+	}
+	host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
+	for _, step := range [][]string{
+		{"make", "build", "SYSTEM=arch"},
+		{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
+	} {
+		cmd := exec.CommandContext(ctx, step[0], step[1:]...)
+		cmd.Dir = tree
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
+		}
+	}
+	lab, err := filepath.Abs("..")
+	if err != nil {
+		return "error", err.Error()
+	}
+	cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
+	cmd.Dir = lab
+	cmd.Env = append(os.Environ(),
+		"MESH_LAB_HOST_BINARY="+host,
+		"MESH_LAB_BOOTSTRAP_BINARY="+installer,
+		// The template the installer knows how to make safe: it rotates the store's and the broker's
+		// well-known credentials, and refuses a template naming neither (the bus one names no broker).
+		"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
+		"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
+		"MESH_REPLAY_CONTROLLER="+controller)
+	out, err := cmd.CombinedOutput()
+	return bedVerdict(string(out), err)
+}
+
+// builderImage is the builder the installer carries, built from the controller at the commit the replay
+// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
+func builderImage(ctx context.Context, controller string) (string, error) {
+	head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
+	if err != nil {
+		return "", err
+	}
+	tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
+	if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
+		return tag, nil
+	}
+	mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
+	if err != nil {
+		return "", err
+	}
+	release := ""
+	for _, line := range strings.Split(string(mod), "\n") {
+		if v, ok := strings.CutPrefix(line, "go "); ok {
+			parts := strings.Split(strings.TrimSpace(v), ".")
+			if len(parts) >= 2 {
+				release = parts[0] + "." + parts[1]
+			}
+		}
+	}
+	if release == "" {
+		return "", errors.New("the controller's go.mod names no Go release to build its builder on")
+	}
+	cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
+		"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
+	cmd.Dir = controller
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
+	}
+	return tag, nil
+}
+
+// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
+// not given what it raises — and never the fault reproduced.
+func bedVerdict(out string, err error) (string, string) {
+	count := func(what string) int {
+		m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
+		if m == nil {
+			return -1
+		}
+		n, _ := strconv.Atoi(m[1])
+		return n
+	}
+	pass, fail, skipped := count("pass"), count("fail"), count("skipped")
+	switch {
+	case skipped > 0:
+		said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
+		if said != nil {
+			return "error", "the bed skipped: " + said[1]
+		}
+		return "error", "the bed skipped: " + lastLines(out, 6)
+	case fail > 0:
+		if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
+			return "fails", strings.TrimSpace(m[1])
+		}
+		return "fails", lastLines(out, 6)
+	case pass > 0 && err == nil:
+		return "passes", lastLines(out, 3)
+	}
+	return "error", lastLines(out, 6)
+}
+
 func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
 	cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
 	cmd.Env = append(os.Environ(), env)