From d03f87b00dc4c3663a89d109abc6400bc0b79402 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:12:03 +0200 Subject: [PATCH 1/2] Register R92 as a bed: genesis on a runtime that trusts no registry (hq issue 92) The lab's base image trusts the documentation ranges as plain-HTTP registries, so no genesis bed could see the push to an untrusted registry that issue 92 is about (reviews 73 and 76 on mesh-host #91). The bed takes that trust off the anchor before the installer runs and asks the runtime it trusts loopback only, then runs genesis with --registry 192.0.2.10:5000 and holds that the control plane reaches the registry, trusted by the mesh's name too. The prover gains a bed kind: it builds the node-engine and its installer at each commit, the carried builder from the pinned controller, and raises the bed; a bed that never reached the step it replays skips, and a skip is an error, never the fault reproduced. Not proved yet: genesis stops at step 6 on both sides, because the temporary controller has no MESH_BUS_NATS and the bus template is refused by the installer (issue 146). --- replays/cmd/prove/main.go | 145 +++++++++++++++++- replays/register.go | 27 ++++ replays/register_test.go | 9 ++ scenarios/genesis-untrusted.yml | 42 ++++++ test/integration/replay92.test.ts | 238 ++++++++++++++++++++++++++++++ 5 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 scenarios/genesis-untrusted.yml create mode 100644 test/integration/replay92.test.ts diff --git a/replays/cmd/prove/main.go b/replays/cmd/prove/main.go index d8ffa73..4cfa578 100644 --- a/replays/cmd/prove/main.go +++ b/replays/cmd/prove/main.go @@ -7,7 +7,9 @@ // The clones are the core repositories, side by side as the lab expects them. A replay in its repository // is laid over a worktree of each commit and run with go test; the bus replay runs against a server of // the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's -// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's. +// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its +// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed +// through to the controller's. package main import ( @@ -20,6 +22,8 @@ import ( "os" "os/exec" "path/filepath" + "regexp" + "strconv" "strings" "time" @@ -63,7 +67,12 @@ func main() { // run is the replay at one commit: passes, fails, or absent (the check it replays did not exist). func run(r replays.Replay, repos, at string, fixed bool) (string, string) { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + limit := 15 * time.Minute + if r.Kind == replays.Bed { + // Two machines raised, a foundation pulled and a control plane built inside one of them. + limit = 90 * time.Minute + } + ctx, cancel := context.WithTimeout(context.Background(), limit) defer cancel() repo := filepath.Join(repos, r.Repository) tree, err := os.MkdirTemp("", "prove-"+r.ID+"-") @@ -77,6 +86,8 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) { defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run() switch r.Kind { + case replays.Bed: + return aBed(ctx, r, repos, tree, fixed) case replays.InRepository, replays.Gate, replays.Liveness: home := filepath.Join(repos, r.Repository) if r.Home != "" { @@ -155,6 +166,136 @@ func run(r replays.Replay, repos, at string, fixed bool) (string, string) { return "error", "no such kind" } +// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its +// installer built from it, the installer carrying a builder built from the controller the replay pins, and +// the bed run with them, the pinned controller to build from and the pinned catalogue to read. +func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) { + beside := map[string]string{} + for _, also := range r.With { + ref := also.Fix + if !fixed { + ref = also.Fix + "^1" + if also.Before != "" { + ref = also.Before + } + } + other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-") + if err != nil { + return "error", err.Error() + } + defer os.RemoveAll(other) + clone := filepath.Join(repos, also.Repository) + if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other, + ref).CombinedOutput(); err != nil { + return "error", string(out) + } + defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run() + beside[also.Repository] = other + } + controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"] + if controller == "" || catalogue == "" { + return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " + + "not both" + } + builder, err := builderImage(ctx, controller) + if err != nil { + return "error", err.Error() + } + host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap") + for _, step := range [][]string{ + {"make", "build", "SYSTEM=arch"}, + {"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer}, + } { + cmd := exec.CommandContext(ctx, step[0], step[1:]...) + cmd.Dir = tree + if out, err := cmd.CombinedOutput(); err != nil { + return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6)) + } + } + lab, err := filepath.Abs("..") + if err != nil { + return "error", err.Error() + } + cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test) + cmd.Dir = lab + cmd.Env = append(os.Environ(), + "MESH_LAB_HOST_BINARY="+host, + "MESH_LAB_BOOTSTRAP_BINARY="+installer, + // The template the installer knows how to make safe: it rotates the store's and the broker's + // well-known credentials, and refuses a template naming neither (the bus one names no broker). + "MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"), + "MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"), + "MESH_REPLAY_CONTROLLER="+controller) + out, err := cmd.CombinedOutput() + return bedVerdict(string(out), err) +} + +// builderImage is the builder the installer carries, built from the controller at the commit the replay +// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit. +func builderImage(ctx context.Context, controller string) (string, error) { + head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output() + if err != nil { + return "", err + } + tag := "mesh-builder:prove-" + strings.TrimSpace(string(head)) + if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil { + return tag, nil + } + mod, err := os.ReadFile(filepath.Join(controller, "go.mod")) + if err != nil { + return "", err + } + release := "" + for _, line := range strings.Split(string(mod), "\n") { + if v, ok := strings.CutPrefix(line, "go "); ok { + parts := strings.Split(strings.TrimSpace(v), ".") + if len(parts) >= 2 { + release = parts[0] + "." + parts[1] + } + } + } + if release == "" { + return "", errors.New("the controller's go.mod names no Go release to build its builder on") + } + cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine", + "-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".") + cmd.Dir = controller + if out, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6)) + } + return tag, nil +} + +// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was +// not given what it raises — and never the fault reproduced. +func bedVerdict(out string, err error) (string, string) { + count := func(what string) int { + m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out) + if m == nil { + return -1 + } + n, _ := strconv.Atoi(m[1]) + return n + } + pass, fail, skipped := count("pass"), count("fail"), count("skipped") + switch { + case skipped > 0: + said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out) + if said != nil { + return "error", "the bed skipped: " + said[1] + } + return "error", "the bed skipped: " + lastLines(out, 6) + case fail > 0: + if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil { + return "fails", strings.TrimSpace(m[1]) + } + return "fails", lastLines(out, 6) + case pass > 0 && err == nil: + return "passes", lastLines(out, 3) + } + return "error", lastLines(out, 6) +} + func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) { cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package) cmd.Env = append(os.Environ(), env) diff --git a/replays/register.go b/replays/register.go index f404812..1af0f79 100644 --- a/replays/register.go +++ b/replays/register.go @@ -32,6 +32,11 @@ const ( // judged by the node-engine at the commit, and the controller at the commit judging its gate from what // the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With. Liveness Kind = "liveness" + // Bed is a bed of the lab, raised on its hypervisor: Test is its file under test/integration, run with + // the node-engine and its installer built from Repository at the commit, beside the With repositories at + // theirs (novox/hq issue 92). A bed that could not reach the step it replays skips, and the prover reads + // a skip as an error, never as the fault reproduced. + Bed Kind = "bed" ) // Also is another repository a replay runs at its own commits beside Repository's: before its fix and on @@ -357,6 +362,28 @@ var Register = []Replay{ "answer: only what needs pacman fails, with the probe's words, and the file is written and recorded", Package: "./cmd/mesh-host", Test: "TestReplay233", Files: []string{"cmd/mesh-host/replay233_test.go"}}, + // Genesis pushed to a registry its runtime did not yet trust (2026-09-22): the first real genesis raised the + // mesh's registry and pushed the control plane's image to it by the address it was given, and the runtime + // refused plain HTTP, because the trust was written by the docker module six steps after the push. The + // lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here, + // whose anchor has that trust taken off before the installer runs, raised with the node-engine and its + // installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned, + // the same on both sides. Fix is the merge of its pull request (mesh-host #91), proved against its first + // parent. + {ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "dd9ab7927d967e236ef755bbe5fd6238d68f10df", + What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " + + "refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " + + "by the docker module, six steps after the push", + Asserts: "on a runtime that trusts no registry but loopback, genesis given --registry :5000 " + + "pushes the control plane to the registry it raised, and the runtime trusts that registry by the " + + "mesh's name too", + Test: "test/integration/replay92.test.ts", + With: []Also{ + {Repository: "mesh-controller", Fix: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1", + Before: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1"}, + {Repository: "mesh-catalog", Fix: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871", + Before: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871"}, + }}, // Not a core incident, and the first of its kind: a module's. The crash loop was found by a person // reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR // 0240 makes the node-engine judge it and the gate fail it. diff --git a/replays/register_test.go b/replays/register_test.go index 393af25..eebab25 100644 --- a/replays/register_test.go +++ b/replays/register_test.go @@ -2,6 +2,7 @@ package replays import ( "regexp" + "slices" "testing" ) @@ -29,6 +30,14 @@ func TestEveryReplayIsWhole(t *testing.T) { if len(r.Files) == 0 || len(r.With) == 0 { t.Errorf("%s runs two repositories and lays the controller's half over its commit: %+v", r.ID, r) } + case Bed: + if !regexp.MustCompile(`^test/integration/[a-z0-9-]+\.test\.ts$`).MatchString(r.Test) { + t.Errorf("%s is a bed and names no bed of the lab to raise: %q", r.ID, r.Test) + } + if !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-controller" }) || + !slices.ContainsFunc(r.With, func(a Also) bool { return a.Repository == "mesh-catalog" }) { + t.Errorf("%s is a bed and does not pin the controller and the catalogue it raises: %+v", r.ID, r) + } case Bus, Resolver: default: t.Errorf("%s is of no kind the prover runs: %q", r.ID, r.Kind) diff --git a/scenarios/genesis-untrusted.yml b/scenarios/genesis-untrusted.yml new file mode 100644 index 0000000..5ac6dcc --- /dev/null +++ b/scenarios/genesis-untrusted.yml @@ -0,0 +1,42 @@ +# GENESIS on a runtime nobody told to trust a registry: the bed of replay R92 (novox/hq issue 92). +# +# The first real genesis pushed the control plane's image to the registry it had just raised, and the +# container runtime refused: "server gave HTTP response to HTTPS client". The mesh's registry speaks plain +# HTTP on purpose, and the trust that lets a runtime speak to it was written six steps after the push. The +# lab never saw it, because its base image trusts the documentation ranges as plain-HTTP registries before +# any bed starts (src/lifecycle/base.ts). The bed takes that trust back off the anchor before genesis runs, +# so the anchor's runtime is a bare machine's: it trusts loopback and nothing else. +# +# Two machines on one public segment, both with a way out — the installer pulls the store, the bus and the +# registry over it, as a bare machine does: +# - anchor: the machine genesis raises, given its registry at its own address, 192.0.2.10:5000; +# - forge: serves the control plane's repository at the commit under test over git's own protocol, so the +# installer builds exactly that and nothing on the workstation listens for the lab. On the anchor +# itself it would be a listener, and genesis refuses a machine something already listens on. +scenario: genesis-untrusted + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + # Genesis-single's anchor: the foundation, the registry, and two control planes during the pivot. + memory: 8GiB + cpus: 4 + disk: 40GiB + + forge: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 2GiB + cpus: 2 + disk: 15GiB + +place: + all: [host, runtime] diff --git a/test/integration/replay92.test.ts b/test/integration/replay92.test.ts new file mode 100644 index 0000000..23870d6 --- /dev/null +++ b/test/integration/replay92.test.ts @@ -0,0 +1,238 @@ +/** + * REPLAY R92 — genesis pushes to its own registry on a runtime nobody told to trust it (novox/hq issue 92). + * + * On the first real genesis the push of the control plane's image into the registry genesis had just + * raised was refused: `Get "https://
/v2/": http: server gave HTTP response to HTTPS client`. The + * registry speaks plain HTTP on purpose (ADR 0082), and the trust that lets a runtime speak to it was + * written by the `docker` module, installed six steps after the push. The fix (mesh-host #91) has genesis + * write that trust into the runtime's configuration itself, and confirm it through `docker info`, before + * anything is pushed. + * + * **The lab hid it.** Its base image trusts the documentation ranges as plain-HTTP registries before any bed + * starts (src/lifecycle/base.ts), so every genesis bed pushed to a registry its runtime already trusted. This + * bed takes that trust back off the anchor — the base's file removed, the runtime restarted, and the runtime + * asked that it now trusts loopback and nothing else — and then runs the installer as an operator does, with + * `--registry 192.0.2.10:5000`. + * + * What it holds: genesis gets past its publish step, the registry it raised serves the control plane's image + * it pushed by the address it was given, and the runtime trusts that registry by the mesh's name too, which + * the builder pushes to later. Before the fix the push is refused, and the bed fails naming it. + * + * **A bed that could not reach the step it replays says so and does not fail**: a genesis stopped before + * its registry step — a pull the internet refused, a build that broke — proves nothing about the order this + * replays, and the test is skipped with the reason, which the prover reads as an error, never as the fault + * reproduced. Genesis is run once and not asked to finish: the steps after the control plane read + * repositories this bed does not serve, and are not what it replays. + * + * MESH_LAB_HOST_BINARY=/mesh-host MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap + * MESH_LAB_BUNDLE=/examples/foundation-first-node.lock (the installer refuses the bus one) + * MESH_LAB_CATALOG=/modules MESH_REPLAY_CONTROLLER= + * MESH_LAB_KEEP=1 leave it standing + * + * Run by `go run ./replays/cmd/prove R92`, which builds all of that at the fix and at the commit before it. + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec, push } from "../../src/lifecycle/operate.ts"; +import { bootstrapBinaryPath, hostBinaryPath } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts"; +import { genesis, type GenesisResult } from "./genesis.ts"; + +const SCENARIO = "genesis-untrusted"; +const ANCHOR = "anchor"; +const FORGE = "forge"; +const ANCHOR_ADDRESS = "192.0.2.10"; +const FORGE_ADDRESS = "192.0.2.30"; +/** The registry as the operator gives it: the anchor's own address and the registry's port. */ +const REGISTRY = `${ANCHOR_ADDRESS}:5000`; +/** And as the mesh names it: the node, the mesh's internal suffix, the same port. */ +const REGISTRY_BY_NAME = `${ANCHOR}.internal:5000`; +/** The installer's steps up to and including the one this replays, in its order. */ +const UP_TO_PUBLISH = ["preflight", "load", "build", "bundle", "apply", "verify", "enrol", "operator", "registry", "publish"]; +/** The steps before the registry's: a stop there never reached what this replays. */ +const BEFORE_REGISTRY = UP_TO_PUBLISH.slice(0, UP_TO_PUBLISH.indexOf("registry")); + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const catalogDir = process.env["MESH_LAB_CATALOG"] ?? ""; +const controller = process.env["MESH_REPLAY_CONTROLLER"] ?? ""; +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "genesis-untrusted-live" : undefined); + +const skip = + !capability.usable ? capability.why : + !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : + !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" : + !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" : + !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" : + !controller || !existsSync(resolve(controller, ".git")) ? "MESH_REPLAY_CONTROLLER is not set to a mesh-controller checkout" : + false; + +let instanceId = ""; +/** Why the bed never reached the step it replays, or "" when it did. */ +let unreached = ""; +let result: GenesisResult | undefined; +let untrusted = ""; +const report: string[] = []; + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} + +/** The control plane's repository at the checkout's HEAD, served from the forge machine over git's protocol. */ +async function raiseForge(): Promise { + const dir = mkdtempSync(join(tmpdir(), "mesh-lab-r92-forge-")); + const head = execFileSync("git", ["-C", controller, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + try { + const bare = join(dir, "mesh-controller.git"); + execFileSync("git", ["init", "-q", "--bare", bare]); + // No collection after the push: one run in the background rewrites the objects while they are carried. + execFileSync("git", ["-C", bare, "config", "receive.autogc", "false"]); + execFileSync("git", ["-C", controller, "push", "-q", "--force", bare, "HEAD:refs/heads/main"], { stdio: "pipe" }); + const tar = join(tmpdir(), `mesh-lab-r92-forge-${process.pid}.tar`); + execFileSync("tar", ["-cf", tar, "-C", dir, "."]); + await push(instanceId, FORGE, tar, "/tmp/forge.tar"); + rmSync(tar, { force: true }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + // Owned by the daemon's user, or git refuses to serve it ("dubious ownership") and says nothing. + await must(FORGE, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` + + `git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE_ADDRESS} --pid-file=/run/git-daemon.pid`); + for (let i = 0; ; i++) { + if ((await on(ANCHOR, `git ls-remote git://${FORGE_ADDRESS}/mesh-controller.git main`)).ok) break; + if (i >= 30) throw new Error(`the anchor cannot reach the forge at git://${FORGE_ADDRESS}/`); + await new Promise((r) => setTimeout(r, 2_000)); + } + return head; +} + +/** + * The anchor's runtime made a bare machine's: the base's trust of the documentation ranges taken off, and + * the runtime asked that it trusts no registry but loopback. Before genesis, so nothing runs that a restart + * would stop. Answers what the runtime said. + */ +async function untrust(): Promise { + await must(ANCHOR, `rm -f /etc/docker/daemon.json && systemctl restart docker`, 180_000); + for (let i = 0; ; i++) { + if ((await on(ANCHOR, `docker info >/dev/null`)).ok) break; + if (i >= 60) throw new Error("the anchor's runtime did not answer after its trust was taken off"); + await new Promise((r) => setTimeout(r, 2_000)); + } + const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig}}'`)).trim(); + const parsed = JSON.parse(config) as { InsecureRegistryCIDRs?: string[]; IndexConfigs?: Record }; + const cidrs = parsed.InsecureRegistryCIDRs ?? []; + const insecureNames = Object.entries(parsed.IndexConfigs ?? {}).filter(([, c]) => c.Secure === false).map(([n]) => n); + if (cidrs.some((c) => c !== "127.0.0.0/8" && c !== "::1/128") || insecureNames.length > 0) { + throw new Error(`the anchor's runtime still trusts more than loopback after the base's trust was taken off: ${config}`); + } + return `trusts ${cidrs.join(", ") || "nothing"} as plain-HTTP registries, and no registry by name`; +} + +before(async () => { + if (skip) return; + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), + }); + instanceId = raised.instanceId; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); + + let head = ""; + try { + head = await raiseForge(); + report.push(` forge git://${FORGE_ADDRESS}/mesh-controller.git at ${head.slice(0, 8)}`); + untrusted = await untrust(); + report.push(` anchor's runtime ${untrusted}`); + } catch (err) { + unreached = `the bed was not prepared, so the installer never ran: ${(err as Error).message}`; + return; + } + + const forge = (repo: string) => `git://${FORGE_ADDRESS}/${repo}.git`; + try { + result = await genesis({ + instanceId, + node: ANCHOR, + installer: installer as string, + catalogDir, + // The registry's manifest is all the steps this replays read from the catalogue. + catalogueModules: ["distribution"], + bundleTemplate: foundationBundle(bundle, []), + registry: REGISTRY, + source: forge("mesh-controller"), + sourceRef: head, + // Named because the installer requires them; served by nobody, since the steps that clone them + // come after the one replayed here. + toolsSource: forge("mesh-tools"), + sdkSource: forge("mesh-sdk"), + catalogSource: forge("mesh-catalog"), + attempts: 1, + verify: false, + log: (m) => console.log(m), + }); + } catch (err) { + unreached = `the installer could not be run: ${(err as Error).message}`; + return; + } + report.push(...result.report); + const stoppedAt = result.step.replace(/^step \d+ of \d+, /, ""); + if (!result.ok && (BEFORE_REGISTRY.includes(stoppedAt) || !result.step)) { + unreached = `genesis stopped at ${result.step || "no step it named"}, before its registry step, so the order ` + + `this replays was never reached:\n${result.why}`; + } +}, { timeout: 3_600_000 }); + +after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 900_000 }); + +test("genesis pushes to the registry it raised on a runtime nobody told to trust it", { + skip, timeout: 3_600_000, +}, async (t) => { + console.log(report.join("\n")); + if (unreached) { + t.skip(`R92 NOT REACHED: ${unreached}`); + return; + } + const ran = result as GenesisResult; + const stoppedAt = ran.step.replace(/^step \d+ of \d+, /, ""); + + // The incident's own words, and the step it stopped at. + assert.doesNotMatch(ran.said, /server gave HTTP response to HTTPS client/, + `the runtime refused to push to ${REGISTRY} over plain HTTP — the registry was not trusted before the push ` + + `(stopped at ${ran.step || "no step"}):\n${ran.why}`); + assert.ok(ran.ok || !UP_TO_PUBLISH.includes(stoppedAt), + `genesis stopped at ${ran.step}, before its control plane was published to ${REGISTRY}:\n${ran.why}`); + + // The registry agreeing: the image genesis pushed, by the address it was given. + const tags = (await on(ANCHOR, `curl -s --max-time 10 http://${REGISTRY}/v2/mesh-controller/tags/list`)).out.trim(); + assert.match(tags, /"genesis"/, `${REGISTRY} does not serve the control plane genesis pushed: ${tags || "no answer"}`); + + // And the runtime trusts the registry by the mesh's name as well: the builder pushes there later. + const config = (await must(ANCHOR, `docker info --format '{{json .RegistryConfig.IndexConfigs}}'`)).trim(); + const byName = (JSON.parse(config) as Record)[REGISTRY_BY_NAME]; + assert.ok(byName && byName.Secure === false, + `the runtime does not trust ${REGISTRY_BY_NAME}, the name the mesh pushes to its registry by: ${config}`); +}); -- 2.54.0 From 0714f7a509332144df61978fd970fb281bb32c3b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 13 Oct 2026 02:59:45 +0200 Subject: [PATCH 2/2] Pin R92 to today's genesis: main with every genesis fix, and the same without #91 (hq issue 92) At the fix's merge no genesis got past step 6 (issue 146), so neither side of the replay reached the push it replays. Fix is mesh-host main 0d11b4fc; Before is that main with #91 reverted, on the unmerged branch replay/r92-before; the controller and catalogue at their mains on both sides. --- replays/register.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/replays/register.go b/replays/register.go index 1af0f79..b61e56d 100644 --- a/replays/register.go +++ b/replays/register.go @@ -368,9 +368,14 @@ var Register = []Replay{ // lab never saw it: its base image trusts the documentation ranges before any bed starts. A bed here, // whose anchor has that trust taken off before the installer runs, raised with the node-engine and its // installer built at the commit; the control plane genesis builds and the catalogue it reads are pinned, - // the same on both sides. Fix is the merge of its pull request (mesh-host #91), proved against its first - // parent. - {ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "dd9ab7927d967e236ef755bbe5fd6238d68f10df", + // the same on both sides. + // + // **Pinned to today's genesis, not to the fix's merge** (hq issues 146, 590, 596, 597): at the merge of + // the fix (mesh-host #91, dd9ab792) no genesis got past step 6, so neither side reached the push. Fix is + // mesh-host main with every genesis fix since; Before is the same main with #91 reverted, kept on the + // branch replay/r92-before and never merged. + {ID: "R92", Issue: 92, Kind: Bed, Repository: "mesh-host", Fix: "0d11b4fc930d6723f3dc54dc173175f66fb6b3b3", + Before: "9e233e1d87569273b4ed353c450b0bd27d21039b", What: "the first real genesis pushed the control plane's image to the registry it had just raised and was " + "refused, server gave HTTP response to HTTPS client: the runtime's trust of the registry was written " + "by the docker module, six steps after the push", @@ -379,10 +384,8 @@ var Register = []Replay{ "mesh's name too", Test: "test/integration/replay92.test.ts", With: []Also{ - {Repository: "mesh-controller", Fix: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1", - Before: "3cfc12167088e098eca7c58d3ff40d3029e1c3d1"}, - {Repository: "mesh-catalog", Fix: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871", - Before: "c5e1bc7bb4228a23957e2524e0ebf387b37a4871"}, + {Repository: "mesh-controller", Fix: "49102e1645c920e42e453caf0aa0746d2b93ce90", Before: "49102e1645c920e42e453caf0aa0746d2b93ce90"}, + {Repository: "mesh-catalog", Fix: "f88e3452f8b8b285ed86e1fe93e1a153f71faada", Before: "f88e3452f8b8b285ed86e1fe93e1a153f71faada"}, }}, // Not a core incident, and the first of its kind: a module's. The crash loop was found by a person // reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR -- 2.54.0