Merge pull request 'The lab runs on the laptop again: the walk's builder on the bus, runtime images on node-tools, enumeration tests independent of the host's incus (hq issues 307, 308)' (#63) from fix/the-lab-runs-on-the-laptop into main

README.md

@@ -198,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a
 says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
 scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
 source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
-beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
-where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
+beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing,
+or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`,
+built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the
+launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs
+`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a
+`MESH_BROKER_URL`, never the served module's own: the runtime refuses that.
 `--no-build` skips this too, and then the bed runs whatever image the store holds.
 
 Check before running a long suite — it says which of these are missing rather than skipping

scenarios/two-nodes.yml

@@ -30,9 +30,9 @@ machines:
     memory: 2GiB
 
 images:
-  # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from
-  # a runtime the mesh builds, so a bed that installs the filter stocks it.
-  - mesh-runtime-nftables:development
+  # No packet-filter runtime: the filter module's verbs are a tools bundle the node's runtime launches
+  # (novox/hq ADR 0170, 0193), not a container of its own, and what this walk needs of the module is
+  # the ruleset it loads — which its resources do. Stocking one built an image nothing ran.
   - mesh-controller:development
   # And the builder, because it is a module the mesh assigns rather than a program somebody
   # starts by hand — which is the only way its credential can be one the mesh delivered.

scripts/build-module-runtime.sh

@@ -1,10 +1,38 @@
 #!/usr/bin/env bash
-# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's
-# compiled code, which serves that module's tools and runs its events/provisioner under the module's
-# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module.
+# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
+# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
 #
 #   build-module-runtime.sh <module> <output.tar>
 #   -> tags mesh-runtime-<module>:development and saves it to <output.tar>
+#
+# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
+# imported every module's compiled code into one process, and this script copied its dist and its
+# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
+# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
+# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
+# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
+# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
+# so the old script failed at its first step.
+#
+# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
+#   1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
+#      entrypoint lands where it is named);
+#   2. writes each entrypoint's launcher, executable;
+#   3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
+#   4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
+#      module's tools, events and provisioner entrypoints, whichever exist.
+#
+# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
+# and is not repeated here.
+#
+# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
+# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
+# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
+# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
+# module's own.
+#
+# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
+# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
 set -euo pipefail
 
 MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
@@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
 MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
 MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
 MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
-MOD="$MESH_CATALOG/modules/$MODULE"
+# Absolute, because the steps below run from inside the module.
+for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
+  [ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
+  printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
+done
+# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
+if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
+MOD="$MESH_CATALOG/$MODULE"
 TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
 BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
+RUNTIME_SRC="$MESH_TOOLS/node-tools"
 [ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
+[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
+command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
 
+# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
+[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
 ( cd "$MESH_SDK" && npm run build >/dev/null )
-( cd "$MESH_TOOLS" && npm run build >/dev/null )
-# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools,
-# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints
-# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage
-# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls
-# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient
-# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the
-# ambient declaration is in the program even though the dep is only installed into the image below.
-SRCS=(); for f in \
-  client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
-  adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \
-  pg.d.ts; do
-  [ -f "$MOD/$f" ] && SRCS+=("$f")
+
+# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
+# builder reads them; a module declaring none falls back to the ones the runtime serves.
+ENTRIES_JS="$(node -e '
+  const m = require(process.argv[1]);
+  const out = new Set();
+  for (const a of (m.build && m.build.artifacts) || [])
+    if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
+  process.stdout.write([...out].join(" "));
+' "$MOD/module.json")"
+if [ -z "$ENTRIES_JS" ]; then
+  for e in tools/index.js index.js provisioner/index.js; do
+    [ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
+  done
+fi
+SRCS=(); for e in $ENTRIES_JS; do
+  [ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
+  SRCS+=("${e%.js}.ts")
 done
+[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
+# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
+# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
+for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
+
 # The module compiles against the SDK, which its package.json names and nothing installs: a module
 # never built on this workstation has no node_modules, and tsc fails on the first import. Installed
-# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
-# the types; the image takes the SDK from MESH_SDK below.
+# as a package copy from the sibling checkout (never a link) when absent.
 if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
   ( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
     || { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
 fi
-# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
-TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
 
 STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
-cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
-cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules"
-# And the sdk, from the sibling this script just built, whatever form the installed tree holds it
-# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised —
-# true only on a workstation where somebody had linked them, and false the moment the runtime's
-# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing
-# compiled in it. The image built then looked fine and every entry point inside it pointed at
-# nothing.
-rm -rf "$STAGE/node_modules/@novox/mesh-sdk"
-mkdir -p "$STAGE/node_modules/@novox"
-cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk"
-rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules"
-# **The image runs compiled code and never compiles any**, so it does not need a compiler. The
-# tree copied above is the runtime's full install, development dependencies and all — and the
-# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing:
-# tsc runs on the workstation a few lines above, not in here.
-#
-# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one
-# of them a git URL with no registry behind it — and could drop something the image needs.
-rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types"
-mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist"
-cp "$MESH_TOOLS/package.json" "$STAGE/package.json"
+DIST="$STAGE/modules/$MODULE/dist"
+# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
+# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
+TSC="$MESH_SDK/node_modules/.bin/tsc"
+( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
+    --rootDir . --outDir "$DIST" 1>&2 )
+# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
+printf '{"type":"module","private":true}\n' > "$DIST/package.json"
+
+# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
+# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
+for e in $ENTRIES_JS; do
+  [ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
+  launcher="$DIST/${e%.js}.serve.mjs"
+  cat > "$launcher" <<LAUNCHER
+#!/usr/bin/env node
+// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
+// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
+import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
+await import("./$(basename "$e")");
+await serveRegisteredOverStdio();
+LAUNCHER
+  chmod 0755 "$launcher"
+done
+
+# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
+# runtime dependencies of its own.
+mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
+cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
+cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
 
 # A module may declare its own third-party runtime deps (the anthropic-manager seals with
-# tweetnacl-sealedbox-js). The shared node_modules copied above carries the common packages and
-# @novox/* — but not a module's private deps. Install those under the module itself, so Node
-# resolves them from /app/modules/<module>/node_modules and still falls back to the shared tree
-# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a
-# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.)
-MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')"
+# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
+# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
+# excluded: there is no public registry for them, and the SDK is staged above.
+MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
 if [ -n "$MOD_DEPS" ]; then
   # shellcheck disable=SC2086
   npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
 fi
 
-# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist.
-ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do
-  [ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e"
+# The runtime itself: node-tools, static, from mesh-tools' Go module.
+( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
+
+# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
+# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
+# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
+# every entrypoint.
+SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
+  [ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
 done
 
 # A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
@@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" <<DOCKER
 FROM $BASE
 WORKDIR /app
 $EXTRA
-COPY package.json ./
+COPY node-tools /usr/local/bin/node-tools
 COPY node_modules ./node_modules
-COPY dist ./dist
 COPY modules ./modules
-ENV MESH_TOOL_MODULES=$ENTRIES
-ENTRYPOINT ["node", "dist/main.js"]
+ENV MESH_TOOL_MODULES=$SERVED
+ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
 DOCKER
 docker build -t "$TAG" "$STAGE"
 docker save -o "$OUT" "$TAG"
-echo "built $TAG (entrypoints: $ENTRIES) -> $OUT"
+echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"

src/incus/client.ts

@@ -20,7 +20,24 @@ import { around, log, shorten } from "../log.ts";
  * predates the group grant cannot reach it — which is a real thing that happens on the
  * machine that just installed it.
  */
-const INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
+let INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
+
+/**
+ * Point every call at another program, and say what it was pointed at before.
+ *
+ * **For a test about how the lab reads a failure, never about the hypervisor.** The command was
+ * read from `MESH_LAB_INCUS` once, when this module loaded — and a test file setting that variable
+ * in its body set it too late, because an ES module's imports are evaluated before its body. So the
+ * enumeration tests asked the real incus wherever one was installed and reachable: they passed on a
+ * machine without incus (the spawn fails) and failed on the workstation that runs the lab, where
+ * listing succeeds. Injected here, a test names the failing program and gets it, on every machine.
+ */
+export function useIncusCommand(argv: string[]): string[] {
+  if (argv.length === 0 || !argv[0]) throw new Error("an incus command needs a program to run");
+  const was = INCUS;
+  INCUS = [...argv];
+  return was;
+}
 
 export interface IncusResult {
   stdout: string;

test/enumerate.test.ts

@@ -1,11 +1,14 @@
-// Set before importing: the client reads MESH_LAB_INCUS once, at module load.
-// `false` is a real program that exits non-zero and prints nothing — which is also the worst
-// case, because an empty stderr is how a failure arrives with no explanation.
-process.env["MESH_LAB_INCUS"] = "false";
-
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { instanceExists, taggedInstances, taggedNetworks } from "../src/incus/client.ts";
+import { instanceExists, taggedInstances, taggedNetworks, useIncusCommand } from "../src/incus/client.ts";
+
+// `false` is a real program that exits non-zero and prints nothing — which is also the worst
+// case, because an empty stderr is how a failure arrives with no explanation.
+//
+// **Injected, never set in the environment here.** This file used to set MESH_LAB_INCUS in its body,
+// which runs after the imports above — so the client had already read the variable, and these tests
+// asked the real incus: green on a machine without one, red on the workstation that runs the lab.
+useIncusCommand(["false"]);
 
 /**
  * "I cannot see" must never be answered as "there is nothing there."

test/integration/harness.ts

@@ -376,9 +376,10 @@ function shellQuote(s: string): string {
  */
 export async function deriveTheFilterOn(o: {
   machine: string; node: string; hubPort: number;
-  /** The images the machines hold. Given, the filter module's runtime — the packet-filter seat's,
-   *  which the mesh would build (novox/hq ADR 0170) — is the stocked one, as for any catalogue
-   *  module a bed installs; the scenario must then stock `mesh-runtime-nftables:development`. */
+  /** The images the machines hold. Given, the manifest is the lab's form of the catalogue's
+   *  (catalogueModule): its build section gone, since the lab builds nothing. The filter's verbs are a
+   *  tools bundle the node's runtime launches (novox/hq ADR 0170, 0193), so no container of the
+   *  module names an image and none need be stocked; what a bed needs of it is the ruleset it loads. */
   held?: HeldImage[];
   must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
   mesh: (command: string, timeoutMs?: number) => Promise<string>;

test/integration/mesh.test.ts

@@ -214,8 +214,33 @@ function tokenFrom(said: string): string {
   return found;
 }
 
-/** The builder started by hand on the anchor, against the foundation broker's plain port on
- *  loopback — the one that builds until a builder module can (see the retired test's note). */
+/**
+ * The build machine's module: a holder of the node-build-agent seat, with the bus credential the mesh
+ * seals to the machine and nothing else.
+ *
+ * **On the bus, the way today's build agent is** (novox/hq ADR 0106, 0127, 0131). The builder used to
+ * be started against the old broker's guest account on loopback; the mesh moved to NATS, the builder
+ * dials only a credential the mesh delivered (`MESH_BROKER_FILE`), and that account exists only as a
+ * module's, composed into the bus's user list. The catalogue's `build-agent` is that module, but it
+ * runs the builder as a container from an artifact the mesh builds and needs the artifact store and
+ * package registry bound; this bed builds nothing before it has a builder, so it declares the same
+ * claim and the same sealed credential and starts the binary it was pointed at itself.
+ */
+const BUILDER_MODULE = "lab-builder";
+const BUILDER_STATE = `/var/lib/mesh/${BUILDER_MODULE}`;
+const BUILDER_MANIFEST = JSON.stringify({
+  module: BUILDER_MODULE, version: "1",
+  claims: [{ name: "node-build-agent", scope: "node", serves: ["current", "kill", "pause", "resume"] }],
+  "own-secrets": { broker: `${BUILDER_STATE}/broker` },
+  resources: [{ id: "mesh-state", type: "directory", path: BUILDER_STATE, mode: "0700" }],
+});
+
+/**
+ * The builder, on the anchor, holding the build seat over the bus with the credential the mesh sealed
+ * to the machine. Assigning the module issues the account (novox/hq issue 203); the bus here is the
+ * bundle's, so its user list is placed by hand (composeTheBusUsers); the push writes the credential
+ * where the module declared it. Idempotent: a warm restore or a second call starts only what is gone.
+ */
 async function startBuilder(): Promise<void> {
   // The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
   // return, so it is pushed whenever the machine has none.
@@ -225,11 +250,31 @@ async function startBuilder(): Promise<void> {
       "--mode", "0755",
     ], 180_000);
   }
+  if (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
+    await must("anchor", `printf %s ${quote(BUILDER_MANIFEST)} > /tmp/${BUILDER_MODULE}.json && ` +
+      `docker cp /tmp/${BUILDER_MODULE}.json mesh-controller:/${BUILDER_MODULE}.json`);
+    await mesh(`module add /${BUILDER_MODULE}.json`);
+    await mesh(`assign anchor ${BUILDER_MODULE}`);
+    await composeTheBusUsers();
+    await mesh("push anchor", 600_000);
+    const by = Date.now() + 180_000;
+    while (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
+      if (Date.now() > by) {
+        assert.fail(`the mesh never delivered the builder's bus credential to ${BUILDER_STATE}/broker:\n` +
+          (await on("anchor", "tail -30 /var/log/mesh-host.log")).out);
+      }
+      await new Promise((r) => setTimeout(r, 3000));
+    }
+  }
   await must("anchor", `mkdir -p /var/lib/mesh-builder`);
   await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
-    `(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
+    `(MESH_BROKER_FILE=${BUILDER_STATE}/broker MESH_NODE=anchor MESH_REGISTRY=${registry} ` +
     `MESH_WORKSPACE=/var/lib/mesh-builder ` +
     `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
+  // Holding the seat, not merely started: a builder refused by the bus exits, and a build asked of a
+  // seat nobody holds waits out its whole timeout before saying anything.
+  const running = await on("anchor", `pgrep -x mesh-builder >/dev/null`);
+  assert.ok(running.ok, `the builder did not stay up:\n${(await on("anchor", "tail -30 /var/log/mesh-builder.log")).out}`);
 }
 
 before(async () => {
@@ -264,8 +309,7 @@ before(async () => {
       const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
       assert.equal(running.out.trim(), "yes",
         "the host did not come back after a restore, so nothing would apply anything");
-      // The hand-started builder is memory too, and the snapshot is disk.
-      if (builder) await startBuilder();
+      // The hand-started builder is memory too; the build test starts it again where it is needed.
 
       console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
         `and started the host again`);
@@ -290,9 +334,8 @@ before(async () => {
   await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
   await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);
 
-  // A build machine, so anything here can ask the mesh to build something. Placed rather than
-  // assumed: nothing else in this scenario would start one.
-  if (builder) await startBuilder();
+  // No build machine yet: the builder holds its seat on a credential the mesh delivers, so it is
+  // started by the test that builds, once the anchor has joined (startBuilder).
   if (warming) {
     // Snapshotted only now, with everything up: a state worth returning to is the one after the
     // part nobody wants to repeat.
@@ -551,11 +594,11 @@ test("a declaration waits for a machine that is switched off", { skip, timeout:
     // Everything needed to tell "the message was never queued" from "the host never read it".
     const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
     const queues = await on("anchor",
-      `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
+      `curl -s 'http://127.0.0.1:8222/jsz?consumers=true' 2>&1 | head -c 4000`);
     const owned = await on("anchor",
       `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
     assert.fail(`a declaration sent to a switched-off machine was lost\n` +
-      `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
+      `--- the host's log ---\n${log.out}\n--- the bus's streams and consumers ---\n${queues.out}\n` +
       `--- what each machine last did ---\n${owned.out}`);
   }
   assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
@@ -1199,9 +1242,8 @@ test("a new commit reaches a machine that is already running the old one", {
   // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
   // question "did my change go out?". This is that question, end to end — a commit, a build, a
   // catalogue, and a machine that ends up running what the source says.
-  // The hand-started builder does not outlive a broker restart, and the foundation's broker is
-  // recreated when the first push reconciles it: a builder is (re)started here, where a build is
-  // asked for. The mesh's own builder is a module with a restart policy and needs none of this.
+  // The builder needs the anchor joined — its credential is the mesh's to deliver — so it is started
+  // here, where a build is asked for. The mesh's own build agent is a module with a restart policy.
   await startBuilder();
   const repo = "/var/lib/mesh/builder/repositories/delivered";
   const write = async (what: string) =>