Register R92 as a bed: genesis on a runtime that trusts no registry (hq issue 92)
mesh/merge-gate out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against a trunk it does not name, the trunk is now 5c1c6f56c9d9; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
The lab's base image trusts the documentation ranges as plain-HTTP registries, so no genesis bed could see the push to an untrusted registry that issue 92 is about (reviews 73 and 76 on mesh-host #91). The bed takes that trust off the anchor before the installer runs and asks the runtime it trusts loopback only, then runs genesis with --registry 192.0.2.10:5000 and holds that the control plane reaches the registry, trusted by the mesh's name too. The prover gains a bed kind: it builds the node-engine and its installer at each commit, the carried builder from the pinned controller, and raises the bed; a bed that never reached the step it replays skips, and a skip is an error, never the fault reproduced. Not proved yet: genesis stops at step 6 on both sides, because the temporary controller has no MESH_BUS_NATS and the bus template is refused by the installer (issue 146).
This commit is contained in:
+143
-2
@@ -7,7 +7,9 @@
|
|||||||
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
|
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
|
||||||
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
|
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
|
||||||
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
|
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
|
||||||
// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's.
|
// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
|
||||||
|
// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
|
||||||
|
// through to the controller's.
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,6 +22,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -63,7 +67,12 @@ func main() {
|
|||||||
|
|
||||||
// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
|
// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
|
||||||
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
limit := 15 * time.Minute
|
||||||
|
if r.Kind == replays.Bed {
|
||||||
|
// Two machines raised, a foundation pulled and a control plane built inside one of them.
|
||||||
|
limit = 90 * time.Minute
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), limit)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
repo := filepath.Join(repos, r.Repository)
|
repo := filepath.Join(repos, r.Repository)
|
||||||
tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
|
tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
|
||||||
@@ -77,6 +86,8 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
|||||||
defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
|
defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()
|
||||||
|
|
||||||
switch r.Kind {
|
switch r.Kind {
|
||||||
|
case replays.Bed:
|
||||||
|
return aBed(ctx, r, repos, tree, fixed)
|
||||||
case replays.InRepository, replays.Gate, replays.Liveness:
|
case replays.InRepository, replays.Gate, replays.Liveness:
|
||||||
home := filepath.Join(repos, r.Repository)
|
home := filepath.Join(repos, r.Repository)
|
||||||
if r.Home != "" {
|
if r.Home != "" {
|
||||||
@@ -155,6 +166,136 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
|
|||||||
return "error", "no such kind"
|
return "error", "no such kind"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
|
||||||
|
// installer built from it, the installer carrying a builder built from the controller the replay pins, and
|
||||||
|
// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
|
||||||
|
func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
|
||||||
|
beside := map[string]string{}
|
||||||
|
for _, also := range r.With {
|
||||||
|
ref := also.Fix
|
||||||
|
if !fixed {
|
||||||
|
ref = also.Fix + "^1"
|
||||||
|
if also.Before != "" {
|
||||||
|
ref = also.Before
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
|
||||||
|
if err != nil {
|
||||||
|
return "error", err.Error()
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(other)
|
||||||
|
clone := filepath.Join(repos, also.Repository)
|
||||||
|
if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
|
||||||
|
ref).CombinedOutput(); err != nil {
|
||||||
|
return "error", string(out)
|
||||||
|
}
|
||||||
|
defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
|
||||||
|
beside[also.Repository] = other
|
||||||
|
}
|
||||||
|
controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
|
||||||
|
if controller == "" || catalogue == "" {
|
||||||
|
return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
|
||||||
|
"not both"
|
||||||
|
}
|
||||||
|
builder, err := builderImage(ctx, controller)
|
||||||
|
if err != nil {
|
||||||
|
return "error", err.Error()
|
||||||
|
}
|
||||||
|
host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
|
||||||
|
for _, step := range [][]string{
|
||||||
|
{"make", "build", "SYSTEM=arch"},
|
||||||
|
{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
|
||||||
|
} {
|
||||||
|
cmd := exec.CommandContext(ctx, step[0], step[1:]...)
|
||||||
|
cmd.Dir = tree
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lab, err := filepath.Abs("..")
|
||||||
|
if err != nil {
|
||||||
|
return "error", err.Error()
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
|
||||||
|
cmd.Dir = lab
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"MESH_LAB_HOST_BINARY="+host,
|
||||||
|
"MESH_LAB_BOOTSTRAP_BINARY="+installer,
|
||||||
|
// The template the installer knows how to make safe: it rotates the store's and the broker's
|
||||||
|
// well-known credentials, and refuses a template naming neither (the bus one names no broker).
|
||||||
|
"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
|
||||||
|
"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
|
||||||
|
"MESH_REPLAY_CONTROLLER="+controller)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return bedVerdict(string(out), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// builderImage is the builder the installer carries, built from the controller at the commit the replay
|
||||||
|
// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
|
||||||
|
func builderImage(ctx context.Context, controller string) (string, error) {
|
||||||
|
head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
|
||||||
|
if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
|
||||||
|
return tag, nil
|
||||||
|
}
|
||||||
|
mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
release := ""
|
||||||
|
for _, line := range strings.Split(string(mod), "\n") {
|
||||||
|
if v, ok := strings.CutPrefix(line, "go "); ok {
|
||||||
|
parts := strings.Split(strings.TrimSpace(v), ".")
|
||||||
|
if len(parts) >= 2 {
|
||||||
|
release = parts[0] + "." + parts[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if release == "" {
|
||||||
|
return "", errors.New("the controller's go.mod names no Go release to build its builder on")
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
|
||||||
|
"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
|
||||||
|
cmd.Dir = controller
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
|
||||||
|
}
|
||||||
|
return tag, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
|
||||||
|
// not given what it raises — and never the fault reproduced.
|
||||||
|
func bedVerdict(out string, err error) (string, string) {
|
||||||
|
count := func(what string) int {
|
||||||
|
m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
|
||||||
|
if m == nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(m[1])
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
pass, fail, skipped := count("pass"), count("fail"), count("skipped")
|
||||||
|
switch {
|
||||||
|
case skipped > 0:
|
||||||
|
said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
|
||||||
|
if said != nil {
|
||||||
|
return "error", "the bed skipped: " + said[1]
|
||||||
|
}
|
||||||
|
return "error", "the bed skipped: " + lastLines(out, 6)
|
||||||
|
case fail > 0:
|
||||||
|
if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
|
||||||
|
return "fails", strings.TrimSpace(m[1])
|
||||||
|
}
|
||||||
|
return "fails", lastLines(out, 6)
|
||||||
|
case pass > 0 && err == nil:
|
||||||
|
return "passes", lastLines(out, 3)
|
||||||
|
}
|
||||||
|
return "error", lastLines(out, 6)
|
||||||
|
}
|
||||||
|
|
||||||
func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
|
func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
|
||||||
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
|
||||||
cmd.Env = append(os.Environ(), env)
|
cmd.Env = append(os.Environ(), env)
|
||||||
|
|||||||
@@ -32,6 +32,11 @@ const (
|
|||||||
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
|
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
|
||||||
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
|
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
|
||||||
Liveness Kind = "liveness"
|
Liveness Kind = "liveness"
|
||||||
|
// Bed is a bed of the lab, raised on its hypervisor: Test is its file under test/integration, run with
|
||||||
|
// the node-engine and its installer built from Repository at the commit, beside the With repositories at
|
||||||
|
// theirs (novox/hq issue 92). A bed that could not reach the step it replays skips, and the prover reads
|
||||||
|
// a skip as an error, never as the fault reproduced.
|
||||||
|
Bed Kind = "bed"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
|
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
|
||||||
@@ -357,6 +362,28 @@ var Register = []Replay{
|
|||||||
"answer: only what needs pacman fails, with the probe's words, and the file is written and recorded",
|
"answer: only what needs pacman fails, with the probe's words, and the file is written and recorded",
|
||||||
Package: "./cmd/mesh-host", Test: "TestReplay233",
|
Package: "./cmd/mesh-host", Test: "TestReplay233",
|
||||||
Files: []string{"cmd/mesh-host/replay233_test.go"}},
|
Files: []string{"cmd/mesh-host/replay233_test.go"}},
|
||||||
|
// Genesis pushed to a registry its runtime did not yet trust (2026-09-22): the first real genesis raised the
|
||||||
|
// mesh's registry and pushed the control plane's image to it by the address it was given, and the runtime
|
||||||
|
// refused plain HTTP, because the trust was written by the docker module six steps after the push. The
|
||||||
|
// lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here,
|
||||||
|
// whose anchor has that trust taken off before the installer runs, raised with the node-engine and its
|
||||||
|
// installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned,
|
||||||
|
// the same on both sides. Fix is the merge of its pull request (mesh-host #91), proved against its first
|
||||||
|
// parent.
|
||||||
|
{ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "dd9ab7927d967e236ef755bbe5fd6238d68f10df",
|
||||||
|
What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " +
|
||||||
|
"refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " +
|
||||||
|
"by the docker module, six steps after the push",
|
||||||
|
Asserts: "on a runtime that trusts no registry but loopback, genesis given --registry <its own address>:5000 " +
|
||||||
|
"pushes the control plane to the registry it raised, and the runtime trusts that registry by the " +
|
||||||
|
"mesh's name too",
|
||||||
|
Test: "test/integration/replay92.test.ts",
|
||||||
|
With: []Also{
|
||||||
|
{Repository: "mesh-controller", Fix: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1",
|
||||||
|
Before: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1"},
|
||||||
|
{Repository: "mesh-catalog", Fix: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871",
|
||||||
|
Before: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871"},
|
||||||
|
}},
|
||||||
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
|
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
|
||||||
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
|
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
|
||||||
// 0240 makes the node-engine judge it and the gate fail it.
|
// 0240 makes the node-engine judge it and the gate fail it.
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package replays
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,6 +30,14 @@ func TestEveryReplayIsWhole(t *testing.T) {
|
|||||||
if len(r.Files) == 0 || len(r.With) == 0 {
|
if len(r.Files) == 0 || len(r.With) == 0 {
|
||||||
t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
|
t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r)
|
||||||
}
|
}
|
||||||
|
case Bed:
|
||||||
|
if !regexp.MustCompile(`^test/integration/[a-z0-9-]+\.test\.ts$`).MatchString(r.Test) {
|
||||||
|
t.Errorf("%s is a bed and names no bed of the lab to raise: %q", r.ID, r.Test)
|
||||||
|
}
|
||||||
|
if !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-controller" }) ||
|
||||||
|
!slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-catalog" }) {
|
||||||
|
t.Errorf("%s is a bed and does not pin the controller and the catalogue it raises: %+v", r.ID, r)
|
||||||
|
}
|
||||||
case Bus, Resolver:
|
case Bus, Resolver:
|
||||||
default:
|
default:
|
||||||
t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)
|
t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind)
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# GENESIS on a runtime nobody told to trust a registry: the bed of replay R92 (novox/hq issue 92).
|
||||||
|
#
|
||||||
|
# The first real genesis pushed the control plane's image to the registry it had just raised, and the
|
||||||
|
# container runtime refused: "server gave HTTP response to HTTPS client". The mesh's registry speaks plain
|
||||||
|
# HTTP on purpose, and the trust that lets a runtime speak to it was written six steps after the push. The
|
||||||
|
# lab never saw it, because its base image trusts the documentation ranges as plain-HTTP registries before
|
||||||
|
# any bed starts (src/lifecycle/base.ts). The bed takes that trust back off the anchor before genesis runs,
|
||||||
|
# so the anchor's runtime is a bare machine's: it trusts loopback and nothing else.
|
||||||
|
#
|
||||||
|
# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the
|
||||||
|
# registry over it, as a bare machine does:
|
||||||
|
# - anchor: the machine genesis raises, given its registry at its own address, 192.0.2.10:5000;
|
||||||
|
# - forge: serves the control plane's repository at the commit under test over git's own protocol, so the
|
||||||
|
# installer builds exactly that and nothing on the workstation listens for the lab. On the anchor
|
||||||
|
# itself it would be a listener, and genesis refuses a machine something already listens on.
|
||||||
|
scenario: genesis-untrusted
|
||||||
|
|
||||||
|
segments:
|
||||||
|
hosting:
|
||||||
|
kind: public
|
||||||
|
cidr: [192.0.2.0/24]
|
||||||
|
|
||||||
|
machines:
|
||||||
|
anchor:
|
||||||
|
at: { segment: hosting, address: [192.0.2.10] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
# Genesis-single's anchor: the foundation, the registry, and two control planes during the pivot.
|
||||||
|
memory: 8GiB
|
||||||
|
cpus: 4
|
||||||
|
disk: 40GiB
|
||||||
|
|
||||||
|
forge:
|
||||||
|
at: { segment: hosting, address: [192.0.2.30] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 2GiB
|
||||||
|
cpus: 2
|
||||||
|
disk: 15GiB
|
||||||
|
|
||||||
|
place:
|
||||||
|
all: [host, runtime]
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
/**
|
||||||
|
* REPLAY R92 — genesis pushes to its own registry on a runtime nobody told to trust it (novox/hq issue 92).
|
||||||
|
*
|
||||||
|
* On the first real genesis the push of the control plane's image into the registry genesis had just
|
||||||
|
* raised was refused: `Get "https://<address>/v2/": http: server gave HTTP response to HTTPS client`. The
|
||||||
|
* registry speaks plain HTTP on purpose (ADR 0082), and the trust that lets a runtime speak to it was
|
||||||
|
* written by the `docker` module, installed six steps after the push. The fix (mesh-host #91) has genesis
|
||||||
|
* write that trust into the runtime's configuration itself, and confirm it through `docker info`, before
|
||||||
|
* anything is pushed.
|
||||||
|
*
|
||||||
|
* **The lab hid it.** Its base image trusts the documentation ranges as plain-HTTP registries before any bed
|
||||||
|
* starts (src/lifecycle/base.ts), so every genesis bed pushed to a registry its runtime already trusted. This
|
||||||
|
* bed takes that trust back off the anchor — the base's file removed, the runtime restarted, and the runtime
|
||||||
|
* asked that it now trusts loopback and nothing else — and then runs the installer as an operator does, with
|
||||||
|
* `--registry 192.0.2.10:5000`.
|
||||||
|
*
|
||||||
|
* What it holds: genesis gets past its publish step, the registry it raised serves the control plane's image
|
||||||
|
* it pushed by the address it was given, and the runtime trusts that registry by the mesh's name too, which
|
||||||
|
* the builder pushes to later. Before the fix the push is refused, and the bed fails naming it.
|
||||||
|
*
|
||||||
|
* **A bed that could not reach the step it replays says so and does not fail**: a genesis stopped before
|
||||||
|
* its registry step — a pull the internet refused, a build that broke — proves nothing about the order this
|
||||||
|
* replays, and the test is skipped with the reason, which the prover reads as an error, never as the fault
|
||||||
|
* reproduced. Genesis is run once and not asked to finish: the steps after the control plane read
|
||||||
|
* repositories this bed does not serve, and are not what it replays.
|
||||||
|
*
|
||||||
|
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||||
|
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock (the installer refuses the bus one)
|
||||||
|
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_REPLAY_CONTROLLER=<a mesh-controller checkout>
|
||||||
|
* MESH_LAB_KEEP=1 leave it standing
|
||||||
|
*
|
||||||
|
* Run by `go run ./replays/cmd/prove R92`, which builds all of that at the fix and at the commit before it.
|
||||||
|
*/
|
||||||
|
import { test, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { execFileSync } from "node:child_process";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
|
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
||||||
|
import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||||
|
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
||||||
|
import { genesis, type GenesisResult } from "./genesis.ts";
|
||||||
|
|
||||||
|
const SCENARIO = "genesis-untrusted";
|
||||||
|
const ANCHOR = "anchor";
|
||||||
|
const FORGE = "forge";
|
||||||
|
const ANCHOR_ADDRESS = "192.0.2.10";
|
||||||
|
const FORGE_ADDRESS = "192.0.2.30";
|
||||||
|
/** The registry as the operator gives it: the anchor's own address and the registry's port. */
|
||||||
|
const REGISTRY = `${ANCHOR_ADDRESS}:5000`;
|
||||||
|
/** And as the mesh names it: the node, the mesh's internal suffix, the same port. */
|
||||||
|
const REGISTRY_BY_NAME = `${ANCHOR}.internal:5000`;
|
||||||
|
/** The installer's steps up to and including the one this replays, in its order. */
|
||||||
|
const UP_TO_PUBLISH = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry", "publish"];
|
||||||
|
/** The steps before the registry's: a stop there never reached what this replays. */
|
||||||
|
const BEFORE_REGISTRY = UP_TO_PUBLISH.slice(0, UP_TO_PUBLISH.indexOf("registry"));
|
||||||
|
|
||||||
|
const capability = await labIsUsable();
|
||||||
|
const binary = hostBinaryPath();
|
||||||
|
const installer = bootstrapBinaryPath();
|
||||||
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||||
|
const controller = process.env["MESH_REPLAY_CONTROLLER"] ?? "";
|
||||||
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||||
|
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-untrusted-live" : undefined);
|
||||||
|
|
||||||
|
const skip =
|
||||||
|
!capability.usable ? capability.why :
|
||||||
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||||
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||||
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
||||||
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||||
|
!controller || !existsSync(resolve(controller, ".git")) ? "MESH_REPLAY_CONTROLLER is not set to a mesh-controller checkout" :
|
||||||
|
false;
|
||||||
|
|
||||||
|
let instanceId = "";
|
||||||
|
/** Why the bed never reached the step it replays, or "" when it did. */
|
||||||
|
let unreached = "";
|
||||||
|
let result: GenesisResult | undefined;
|
||||||
|
let untrusted = "";
|
||||||
|
const report: string[] = [];
|
||||||
|
|
||||||
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||||
|
const marker = stdout.lastIndexOf("__exit=");
|
||||||
|
if (marker < 0) return { out: stdout, ok: false };
|
||||||
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||||
|
}
|
||||||
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const { out, ok } = await on(machine, command, timeoutMs);
|
||||||
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The control plane's repository at the checkout's HEAD, served from the forge machine over git's protocol. */
|
||||||
|
async function raiseForge(): Promise<string> {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-r92-forge-"));
|
||||||
|
const head = execFileSync("git", ["-C", controller, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||||
|
try {
|
||||||
|
const bare = join(dir, "mesh-controller.git");
|
||||||
|
execFileSync("git", ["init", "-q", "--bare", bare]);
|
||||||
|
// No collection after the push: one run in the background rewrites the objects while they are carried.
|
||||||
|
execFileSync("git", ["-C", bare, "config", "receive.autogc", "false"]);
|
||||||
|
execFileSync("git", ["-C", controller, "push", "-q", "--force", bare, "HEAD:refs/heads/main"], { stdio: "pipe" });
|
||||||
|
const tar = join(tmpdir(), `mesh-lab-r92-forge-${process.pid}.tar`);
|
||||||
|
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||||
|
await push(instanceId, FORGE, tar, "/tmp/forge.tar");
|
||||||
|
rmSync(tar, { force: true });
|
||||||
|
} finally {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
// Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing.
|
||||||
|
await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||||
|
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`);
|
||||||
|
for (let i = 0; ; i++) {
|
||||||
|
if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-controller.git main`)).ok) break;
|
||||||
|
if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`);
|
||||||
|
await new Promise((r) => setTimeout(r, 2_000));
|
||||||
|
}
|
||||||
|
return head;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The anchor's runtime made a bare machine's: the base's trust of the documentation ranges taken off, and
|
||||||
|
* the runtime asked that it trusts no registry but loopback. Before genesis, so nothing runs that a restart
|
||||||
|
* would stop. Answers what the runtime said.
|
||||||
|
*/
|
||||||
|
async function untrust(): Promise<string> {
|
||||||
|
await must(ANCHOR, `rm -f /etc/docker/daemon.json && systemctl restart docker`, 180_000);
|
||||||
|
for (let i = 0; ; i++) {
|
||||||
|
if ((await on(ANCHOR, `docker info >/dev/null`)).ok) break;
|
||||||
|
if (i >= 60) throw new Error("the anchor's runtime did not answer after its trust was taken off");
|
||||||
|
await new Promise((r) => setTimeout(r, 2_000));
|
||||||
|
}
|
||||||
|
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig}}'`)).trim();
|
||||||
|
const parsed = JSON.parse(config) as { InsecureRegistryCIDRs?: string[]; IndexConfigs?: Record<string, { Secure?: boolean }> };
|
||||||
|
const cidrs = parsed.InsecureRegistryCIDRs ?? [];
|
||||||
|
const insecureNames = Object.entries(parsed.IndexConfigs ?? {}).filter(([, c]) => c.Secure === false).map(([n]) => n);
|
||||||
|
if (cidrs.some((c) => c !== "127.0.0.0/8" && c !== "::1/128") || insecureNames.length > 0) {
|
||||||
|
throw new Error(`the anchor's runtime still trusts more than loopback after the base's trust was taken off: ${config}`);
|
||||||
|
}
|
||||||
|
return `trusts ${cidrs.join(", ") || "nothing"} as plain-HTTP registries, and no registry by name`;
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
if (skip) return;
|
||||||
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||||
|
});
|
||||||
|
instanceId = raised.instanceId;
|
||||||
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||||
|
|
||||||
|
let head = "";
|
||||||
|
try {
|
||||||
|
head = await raiseForge();
|
||||||
|
report.push(` forge git://${FORGE_ADDRESS}/mesh-controller.git at ${head.slice(0, 8)}`);
|
||||||
|
untrusted = await untrust();
|
||||||
|
report.push(` anchor's runtime ${untrusted}`);
|
||||||
|
} catch (err) {
|
||||||
|
unreached = `the bed was not prepared, so the installer never ran: ${(err as Error).message}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`;
|
||||||
|
try {
|
||||||
|
result = await genesis({
|
||||||
|
instanceId,
|
||||||
|
node: ANCHOR,
|
||||||
|
installer: installer as string,
|
||||||
|
catalogDir,
|
||||||
|
// The registry's manifest is all the steps this replays read from the catalogue.
|
||||||
|
catalogueModules: ["distribution"],
|
||||||
|
bundleTemplate: foundationBundle(bundle, []),
|
||||||
|
registry: REGISTRY,
|
||||||
|
source: forge("mesh-controller"),
|
||||||
|
sourceRef: head,
|
||||||
|
// Named because the installer requires them; served by nobody, since the steps that clone them
|
||||||
|
// come after the one replayed here.
|
||||||
|
toolsSource: forge("mesh-tools"),
|
||||||
|
sdkSource: forge("mesh-sdk"),
|
||||||
|
catalogSource: forge("mesh-catalog"),
|
||||||
|
attempts: 1,
|
||||||
|
verify: false,
|
||||||
|
log: (m) => console.log(m),
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
unreached = `the installer could not be run: ${(err as Error).message}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
report.push(...result.report);
|
||||||
|
const stoppedAt = result.step.replace(/^step \d+ of \d+, /, "");
|
||||||
|
if (!result.ok && (BEFORE_REGISTRY.includes(stoppedAt) || !result.step)) {
|
||||||
|
unreached = `genesis stopped at ${result.step || "no step it named"}, before its registry step, so the order ` +
|
||||||
|
`this replays was never reached:\n${result.why}`;
|
||||||
|
}
|
||||||
|
}, { timeout: 3_600_000 });
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
if (KEEP) {
|
||||||
|
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (instanceId) await destroy(instanceId);
|
||||||
|
await destroyAll(`${SCENARIO}-`);
|
||||||
|
}, { timeout: 900_000 });
|
||||||
|
|
||||||
|
test("genesis pushes to the registry it raised on a runtime nobody told to trust it", {
|
||||||
|
skip, timeout: 3_600_000,
|
||||||
|
}, async (t) => {
|
||||||
|
console.log(report.join("\n"));
|
||||||
|
if (unreached) {
|
||||||
|
t.skip(`R92 NOT REACHED: ${unreached}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const ran = result as GenesisResult;
|
||||||
|
const stoppedAt = ran.step.replace(/^step \d+ of \d+, /, "");
|
||||||
|
|
||||||
|
// The incident's own words, and the step it stopped at.
|
||||||
|
assert.doesNotMatch(ran.said, /server gave HTTP response to HTTPS client/,
|
||||||
|
`the runtime refused to push to ${REGISTRY} over plain HTTP — the registry was not trusted before the push ` +
|
||||||
|
`(stopped at ${ran.step || "no step"}):\n${ran.why}`);
|
||||||
|
assert.ok(ran.ok || !UP_TO_PUBLISH.includes(stoppedAt),
|
||||||
|
`genesis stopped at ${ran.step}, before its control plane was published to ${REGISTRY}:\n${ran.why}`);
|
||||||
|
|
||||||
|
// The registry agreeing: the image genesis pushed, by the address it was given.
|
||||||
|
const tags = (await on(ANCHOR, `curl -s --max-time 10 http://${REGISTRY}/v2/mesh-controller/tags/list`)).out.trim();
|
||||||
|
assert.match(tags, /"genesis"/, `${REGISTRY} does not serve the control plane genesis pushed: ${tags || "no answer"}`);
|
||||||
|
|
||||||
|
// And the runtime trusts the registry by the mesh's name as well: the builder pushes there later.
|
||||||
|
const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig.IndexConfigs}}'`)).trim();
|
||||||
|
const byName = (JSON.parse(config) as Record<string, { Secure?: boolean }>)[REGISTRY_BY_NAME];
|
||||||
|
assert.ok(byName && byName.Secure === false,
|
||||||
|
`the runtime does not trust ${REGISTRY_BY_NAME}, the name the mesh pushes to its registry by: ${config}`);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user