Register R92 as a bed: genesis on a runtime that trusts no registry (hq issue 92)
mesh/merge-gate out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request

The lab's base image trusts the documentation ranges as plain-HTTP registries, so no genesis bed could
see the push to an untrusted registry that issue 92 is about (reviews 73 and 76 on mesh-host #91). The
bed takes that trust off the anchor before the installer runs and asks the runtime it trusts loopback
only, then runs genesis with --registry 192.0.2.10:5000 and holds that the control plane reaches the
registry, trusted by the mesh's name too.

The prover gains a bed kind: it builds the node-engine and its installer at each commit, the carried
builder from the pinned controller, and raises the bed; a bed that never reached the step it replays
skips, and a skip is an error, never the fault reproduced.

Not proved yet: genesis stops at step 6 on both sides, because the temporary controller has no
MESH_BUS_NATS and the bus template is refused by the installer (issue 146).
This commit is contained in:
jochen
2026-10-12 17:12:03 +02:00
parent 930b13ff47
commit 4d2ee33bd2
5 changed files with 459 additions and 2 deletions
+143 -2
View File
@@ -7,7 +7,9 @@
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's.
// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
// through to the controller's.
package main
import (
@@ -20,6 +22,8 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -63,7 +67,12 @@ func main() {
// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
limit := 15 * time.Minute
if r.Kind == replays.Bed {
// Two machines raised, a foundation pulled and a control plane built inside one of them.
limit = 90 * time.Minute
}
ctx, cancel := context.WithTimeout(context.Background(), limit)
defer cancel()
repo := filepath.Join(repos, r.Repository)
tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
@@ -77,6 +86,8 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
switch r.Kind {
case replays.Bed:
return aBed(ctx, r, repos, tree, fixed)
case replays.InRepository, replays.Gate, replays.Liveness:
home := filepath.Join(repos, r.Repository)
if r.Home != "" {
@@ -155,6 +166,136 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
return "error", "no such kind"
}
// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
// installer built from it, the installer carrying a builder built from the controller the replay pins, and
// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
beside := map[string]string{}
for _, also := range r.With {
ref := also.Fix
if !fixed {
ref = also.Fix + "^1"
if also.Before != "" {
ref = also.Before
}
}
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
if err != nil {
return "error", err.Error()
}
defer os.RemoveAll(other)
clone := filepath.Join(repos, also.Repository)
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
ref).CombinedOutput(); err != nil {
return "error", string(out)
}
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
beside[also.Repository] = other
}
controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
if controller == "" || catalogue == "" {
return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
"not both"
}
builder, err := builderImage(ctx, controller)
if err != nil {
return "error", err.Error()
}
host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
for _, step := range [][]string{
{"make", "build", "SYSTEM=arch"},
{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
} {
cmd := exec.CommandContext(ctx, step[0], step[1:]...)
cmd.Dir = tree
if out, err := cmd.CombinedOutput(); err != nil {
return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
}
}
lab, err := filepath.Abs("..")
if err != nil {
return "error", err.Error()
}
cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
cmd.Dir = lab
cmd.Env = append(os.Environ(),
"MESH_LAB_HOST_BINARY="+host,
"MESH_LAB_BOOTSTRAP_BINARY="+installer,
// The template the installer knows how to make safe: it rotates the store's and the broker's
// well-known credentials, and refuses a template naming neither (the bus one names no broker).
"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
"MESH_REPLAY_CONTROLLER="+controller)
out, err := cmd.CombinedOutput()
return bedVerdict(string(out), err)
}
// builderImage is the builder the installer carries, built from the controller at the commit the replay
// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
func builderImage(ctx context.Context, controller string) (string, error) {
head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
if err != nil {
return "", err
}
tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
return tag, nil
}
mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
if err != nil {
return "", err
}
release := ""
for _, line := range strings.Split(string(mod), "\n") {
if v, ok := strings.CutPrefix(line, "go "); ok {
parts := strings.Split(strings.TrimSpace(v), ".")
if len(parts) >= 2 {
release = parts[0] + "." + parts[1]
}
}
}
if release == "" {
return "", errors.New("the controller's go.mod names no Go release to build its builder on")
}
cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
cmd.Dir = controller
if out, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
}
return tag, nil
}
// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
// not given what it raises — and never the fault reproduced.
func bedVerdict(out string, err error) (string, string) {
count := func(what string) int {
m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
if m == nil {
return -1
}
n, _ := strconv.Atoi(m[1])
return n
}
pass, fail, skipped := count("pass"), count("fail"), count("skipped")
switch {
case skipped > 0:
said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
if said != nil {
return "error", "the bed skipped: " + said[1]
}
return "error", "the bed skipped: " + lastLines(out, 6)
case fail > 0:
if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
return "fails", strings.TrimSpace(m[1])
}
return "fails", lastLines(out, 6)
case pass > 0 && err == nil:
return "passes", lastLines(out, 3)
}
return "error", lastLines(out, 6)
}
func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
cmd.Env = append(os.Environ(), env)
+27
View File
@@ -32,6 +32,11 @@ const (
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
Liveness Kind = "liveness"
// Bed is a bed of the lab, raised on its hypervisor: Test is its file under test/integration, run with
// the node-engine and its installer built from Repository at the commit, beside the With repositories at
// theirs (novox/hq issue 92). A bed that could not reach the step it replays skips, and the prover reads
// a skip as an error, never as the fault reproduced.
Bed Kind = "bed"
)
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
@@ -357,6 +362,28 @@ var Register = []Replay{
"answer: only what needs pacman fails, with the probe's words, and the file is written and recorded",
Package: "./cmd/mesh-host", Test: "TestReplay233",
Files: []string{"cmd/mesh-host/replay233_test.go"}},
// Genesis pushed to a registry its runtime did not yet trust (2026-09-22): the first real genesis raised the
// mesh's registry and pushed the control plane's image to it by the address it was given, and the runtime
// refused plain HTTP, because the trust was written by the docker module six steps after the push. The
// lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here,
// whose anchor has that trust taken off before the installer runs, raised with the node-engine and its
// installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned,
// the same on both sides. Fix is the merge of its pull request (mesh-host #91), proved against its first
// parent.
{ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "dd9ab7927d967e236ef755bbe5fd6238d68f10df",
What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " +
"refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " +
"by the docker module, six steps after the push",
Asserts: "on a runtime that trusts no registry but loopback, genesis given --registry <its own address>:5000 " +
"pushes the control plane to the registry it raised, and the runtime trusts that registry by the " +
"mesh's name too",
Test: "test/integration/replay92.test.ts",
With: []Also{
{Repository: "mesh-controller", Fix: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1",
Before: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1"},
{Repository: "mesh-catalog", Fix: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871",
Before: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871"},
}},
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
// 0240 makes the node-engine judge it and the gate fail it.
+9
View File
@@ -2,6 +2,7 @@ package replays
import (
"regexp"
"slices"
"testing"
)
@@ -29,6 +30,14 @@ func TestEveryReplayIsWhole(t *testing.T) {
if len(r.Files) == 0 || len(r.With) == 0 {
t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
}
case Bed:
if !regexp.MustCompile(`^test/integration/[a-z0-9-]+\.test\.ts$`).MatchString(r.Test) {
t.Errorf("%s is a bed and names no bed of the lab to raise: %q", r.ID, r.Test)
}
if !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-controller" }) ||
!slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-catalog" }) {
t.Errorf("%s is a bed and does not pin the controller and the catalogue it raises: %+v", r.ID, r)
}
case Bus, Resolver:
default:
t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)