diff --git a/README.md b/README.md index 38cbe33..b8c4362 100644 --- a/README.md +++ b/README.md @@ -198,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds' scenarios stock (`mesh-runtime-:development`): each is compared against the module's source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts -beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh` -where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075). +beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing, +or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`, +built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the +launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs +`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a +`MESH_BROKER_URL`, never the served module's own: the runtime refuses that. `--no-build` skips this too, and then the bed runs whatever image the store holds. Check before running a long suite — it says which of these are missing rather than skipping diff --git a/replays/register.go b/replays/register.go index e868ebb..b2bd959 100644 --- a/replays/register.go +++ b/replays/register.go @@ -114,6 +114,25 @@ var Register = []Replay{ Asserts: "a merge adding a module asks the build seat for it, at the branch merged into, and opens no plan", Package: "./cmd/mesh-controller", Test: "TestReplay300", Files: []string{"cmd/mesh-controller/replays_test.go"}, Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"}, + // A drill counted as a repair (2026-10-07): S15 read two deliberate drills, recorded through hand-act + // record for want of a verb, as a cause repaired twice. In a file of its own, since replays_test.go holds + // replays of later commits that do not build at the commit before this fix. Home is the controller's + // commit that added it; it stays reachable once that pull request merges. + {ID: "R292", Issue: 292, Kind: InRepository, Repository: "mesh-controller", Fix: "863ebd4", + What: "two drills of ADR 0240, each with the operator's word, recorded through hand-act record --cause drill, " + + "raised mesh.hand-acts.drill.healer-wanted", + Asserts: "a drill recorded in the hand-act log wants no healer, however it was recorded", + Package: "./cmd/mesh-controller", Test: "TestReplay292", Files: []string{"cmd/mesh-controller/replay292_test.go"}, + Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"}, + // A seat's new verb deadlocked across two repositories (2026-10-07): the controller promising checks + // refused the delivery seat's holder that did not serve it, and a holder serving it was refused by the + // controller that did not promise it. + {ID: "R298", Issue: 298, Kind: InRepository, Repository: "mesh-controller", Fix: "f6aea4b", + What: "the delivery seat's new verb checks could land in neither repository first: each side's claim " + + "check refused the other's holder", + Asserts: "the delivery seat's holder holds it both before and after it serves checks", + Package: "./internal/catalogue", Test: "TestReplay298", Files: []string{"internal/catalogue/replays_test.go"}, + Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"}, // The push a recorded build waits for, counted as a repair (2026-10-07): a test in the controller, in // the fix's own commit, recording two such pushes as the seat's push records them. {ID: "R301", Issue: 301, Kind: InRepository, Repository: "mesh-controller", Fix: "e92a3fe", diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index cc7d769..75f2b8a 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -30,9 +30,9 @@ machines: memory: 2GiB images: - # The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from - # a runtime the mesh builds, so a bed that installs the filter stocks it. - - mesh-runtime-nftables:development + # No packet-filter runtime: the filter module's verbs are a tools bundle the node's runtime launches + # (novox/hq ADR 0170, 0193), not a container of its own, and what this walk needs of the module is + # the ruleset it loads — which its resources do. Stocking one built an image nothing ran. - mesh-controller:development # And the builder, because it is a module the mesh assigns rather than a program somebody # starts by hand — which is the only way its credential can be one the mesh delivered. diff --git a/scripts/build-module-runtime.sh b/scripts/build-module-runtime.sh index 53f57a5..3dbe40a 100755 --- a/scripts/build-module-runtime.sh +++ b/scripts/build-module-runtime.sh @@ -1,10 +1,38 @@ #!/usr/bin/env bash -# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's -# compiled code, which serves that module's tools and runs its events/provisioner under the module's -# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module. +# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built +# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193). # # build-module-runtime.sh # -> tags mesh-runtime-:development and saves it to +# +# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that +# imported every module's compiled code into one process, and this script copied its dist and its +# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/` +# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child +# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes +# beside each compiled entrypoint an executable `.serve.mjs` that imports it and serves what it +# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since, +# so the old script failed at its first step. +# +# This does what the mesh's builder does for a TypeScript bundle, on the workstation: +# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an +# entrypoint lands where it is named); +# 2. writes each entrypoint's launcher, executable; +# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies; +# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the +# module's tools, events and provisioner entrypoints, whichever exist. +# +# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour, +# and is not repeated here. +# +# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it +# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of +# other modules, it is not one). So the container is given the node's runtime credential (the node-tools +# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served +# module's own. +# +# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and +# MESH_CATALOG checkouts. Each defaults to the sibling of this repository. set -euo pipefail MODULE="${1:?usage: build-module-runtime.sh }" @@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)" MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}" MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}" MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}" -MOD="$MESH_CATALOG/modules/$MODULE" +# Absolute, because the steps below run from inside the module. +for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do + [ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; } + printf -v "$v" '%s' "$(cd "${!v}" && pwd)" +done +# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either. +if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi +MOD="$MESH_CATALOG/$MODULE" TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}" BASE="${RUNTIME_BASE:-node:22-bookworm-slim}" +RUNTIME_SRC="$MESH_TOOLS/node-tools" [ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; } +[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; } +command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; } +# The SDK, built: the module compiles against its types and the launchers import its stdio loop. +[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent ) ( cd "$MESH_SDK" && npm run build >/dev/null ) -( cd "$MESH_TOOLS" && npm run build >/dev/null ) -# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools, -# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints -# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage -# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls -# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient -# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the -# ambient declaration is in the program even though the dep is only installed into the image below. -SRCS=(); for f in \ - client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \ - adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \ - pg.d.ts; do - [ -f "$MOD/$f" ] && SRCS+=("$f") + +# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the +# builder reads them; a module declaring none falls back to the ones the runtime serves. +ENTRIES_JS="$(node -e ' + const m = require(process.argv[1]); + const out = new Set(); + for (const a of (m.build && m.build.artifacts) || []) + if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e); + process.stdout.write([...out].join(" ")); +' "$MOD/module.json")" +if [ -z "$ENTRIES_JS" ]; then + for e in tools/index.js index.js provisioner/index.js; do + [ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e" + done +fi +SRCS=(); for e in $ENTRIES_JS; do + [ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; } + SRCS+=("${e%.js}.ts") done +[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; } +# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's +# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image. +for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done + # The module compiles against the SDK, which its package.json names and nothing installs: a module # never built on this workstation has no node_modules, and tsc fails on the first import. Installed -# as a package copy from the sibling checkout (never a link) when absent — the compile needs only -# the types; the image takes the SDK from MESH_SDK below. +# as a package copy from the sibling checkout (never a link) when absent. if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then ( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \ || { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; } fi -# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. -TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 ) STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT -cp -r "$MESH_TOOLS/dist" "$STAGE/dist" -cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules" -# And the sdk, from the sibling this script just built, whatever form the installed tree holds it -# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised — -# true only on a workstation where somebody had linked them, and false the moment the runtime's -# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing -# compiled in it. The image built then looked fine and every entry point inside it pointed at -# nothing. -rm -rf "$STAGE/node_modules/@novox/mesh-sdk" -mkdir -p "$STAGE/node_modules/@novox" -cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk" -rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules" -# **The image runs compiled code and never compiles any**, so it does not need a compiler. The -# tree copied above is the runtime's full install, development dependencies and all — and the -# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing: -# tsc runs on the workstation a few lines above, not in here. -# -# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one -# of them a git URL with no registry behind it — and could drop something the image needs. -rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types" -mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist" -cp "$MESH_TOOLS/package.json" "$STAGE/package.json" +DIST="$STAGE/modules/$MODULE/dist" +# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at +# the module, as the builder compiles, so tools/index.ts lands at tools/index.js. +TSC="$MESH_SDK/node_modules/.bin/tsc" +( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \ + --rootDir . --outDir "$DIST" 1>&2 ) +# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does. +printf '{"type":"module","private":true}\n' > "$DIST/package.json" + +# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder, +# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio. +for e in $ENTRIES_JS; do + [ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; } + launcher="$DIST/${e%.js}.serve.mjs" + cat > "$launcher" </node_modules and still falls back to the shared tree -# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a -# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.) -MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')" +# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from +# /app/modules//node_modules and still falls back to /app/node_modules for the SDK. @novox/* are +# excluded: there is no public registry for them, and the SDK is staged above. +MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")" if [ -n "$MOD_DEPS" ]; then # shellcheck disable=SC2086 npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null fi -# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist. -ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do - [ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e" +# The runtime itself: node-tools, static, from mesh-tools' Go module. +( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools ) + +# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the +# module has — each by its launcher, as =. The others (bootstrap, prepare, apply, …) are +# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for +# every entrypoint. +SERVED=""; for e in tools/index.js index.js provisioner/index.js; do + [ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs" done # A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio @@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" < $OUT" +echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT" diff --git a/src/incus/client.ts b/src/incus/client.ts index 31d9e66..276b63f 100644 --- a/src/incus/client.ts +++ b/src/incus/client.ts @@ -20,7 +20,24 @@ import { around, log, shorten } from "../log.ts"; * predates the group grant cannot reach it — which is a real thing that happens on the * machine that just installed it. */ -const INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean); +let INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean); + +/** + * Point every call at another program, and say what it was pointed at before. + * + * **For a test about how the lab reads a failure, never about the hypervisor.** The command was + * read from `MESH_LAB_INCUS` once, when this module loaded — and a test file setting that variable + * in its body set it too late, because an ES module's imports are evaluated before its body. So the + * enumeration tests asked the real incus wherever one was installed and reachable: they passed on a + * machine without incus (the spawn fails) and failed on the workstation that runs the lab, where + * listing succeeds. Injected here, a test names the failing program and gets it, on every machine. + */ +export function useIncusCommand(argv: string[]): string[] { + if (argv.length === 0 || !argv[0]) throw new Error("an incus command needs a program to run"); + const was = INCUS; + INCUS = [...argv]; + return was; +} export interface IncusResult { stdout: string; diff --git a/test/enumerate.test.ts b/test/enumerate.test.ts index 93a8b4d..ee78b51 100644 --- a/test/enumerate.test.ts +++ b/test/enumerate.test.ts @@ -1,11 +1,14 @@ -// Set before importing: the client reads MESH_LAB_INCUS once, at module load. -// `false` is a real program that exits non-zero and prints nothing — which is also the worst -// case, because an empty stderr is how a failure arrives with no explanation. -process.env["MESH_LAB_INCUS"] = "false"; - import { test } from "node:test"; import assert from "node:assert/strict"; -import { instanceExists, taggedInstances, taggedNetworks } from "../src/incus/client.ts"; +import { instanceExists, taggedInstances, taggedNetworks, useIncusCommand } from "../src/incus/client.ts"; + +// `false` is a real program that exits non-zero and prints nothing — which is also the worst +// case, because an empty stderr is how a failure arrives with no explanation. +// +// **Injected, never set in the environment here.** This file used to set MESH_LAB_INCUS in its body, +// which runs after the imports above — so the client had already read the variable, and these tests +// asked the real incus: green on a machine without one, red on the workstation that runs the lab. +useIncusCommand(["false"]); /** * "I cannot see" must never be answered as "there is nothing there." diff --git a/test/integration/harness.ts b/test/integration/harness.ts index da8e458..5c9b65d 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -376,9 +376,10 @@ function shellQuote(s: string): string { */ export async function deriveTheFilterOn(o: { machine: string; node: string; hubPort: number; - /** The images the machines hold. Given, the filter module's runtime — the packet-filter seat's, - * which the mesh would build (novox/hq ADR 0170) — is the stocked one, as for any catalogue - * module a bed installs; the scenario must then stock `mesh-runtime-nftables:development`. */ + /** The images the machines hold. Given, the manifest is the lab's form of the catalogue's + * (catalogueModule): its build section gone, since the lab builds nothing. The filter's verbs are a + * tools bundle the node's runtime launches (novox/hq ADR 0170, 0193), so no container of the + * module names an image and none need be stocked; what a bed needs of it is the ruleset it loads. */ held?: HeldImage[]; must: (machine: string, command: string, timeoutMs?: number) => Promise; mesh: (command: string, timeoutMs?: number) => Promise; diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index ccad1ac..269f635 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -214,8 +214,33 @@ function tokenFrom(said: string): string { return found; } -/** The builder started by hand on the anchor, against the foundation broker's plain port on - * loopback — the one that builds until a builder module can (see the retired test's note). */ +/** + * The build machine's module: a holder of the node-build-agent seat, with the bus credential the mesh + * seals to the machine and nothing else. + * + * **On the bus, the way today's build agent is** (novox/hq ADR 0106, 0127, 0131). The builder used to + * be started against the old broker's guest account on loopback; the mesh moved to NATS, the builder + * dials only a credential the mesh delivered (`MESH_BROKER_FILE`), and that account exists only as a + * module's, composed into the bus's user list. The catalogue's `build-agent` is that module, but it + * runs the builder as a container from an artifact the mesh builds and needs the artifact store and + * package registry bound; this bed builds nothing before it has a builder, so it declares the same + * claim and the same sealed credential and starts the binary it was pointed at itself. + */ +const BUILDER_MODULE = "lab-builder"; +const BUILDER_STATE = `/var/lib/mesh/${BUILDER_MODULE}`; +const BUILDER_MANIFEST = JSON.stringify({ + module: BUILDER_MODULE, version: "1", + claims: [{ name: "node-build-agent", scope: "node", serves: ["current", "kill", "pause", "resume"] }], + "own-secrets": { broker: `${BUILDER_STATE}/broker` }, + resources: [{ id: "mesh-state", type: "directory", path: BUILDER_STATE, mode: "0700" }], +}); + +/** + * The builder, on the anchor, holding the build seat over the bus with the credential the mesh sealed + * to the machine. Assigning the module issues the account (novox/hq issue 203); the bus here is the + * bundle's, so its user list is placed by hand (composeTheBusUsers); the push writes the credential + * where the module declared it. Idempotent: a warm restore or a second call starts only what is gone. + */ async function startBuilder(): Promise { // The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a // return, so it is pushed whenever the machine has none. @@ -225,11 +250,31 @@ async function startBuilder(): Promise { "--mode", "0755", ], 180_000); } + if (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) { + await must("anchor", `printf %s ${quote(BUILDER_MANIFEST)} > /tmp/${BUILDER_MODULE}.json && ` + + `docker cp /tmp/${BUILDER_MODULE}.json mesh-controller:/${BUILDER_MODULE}.json`); + await mesh(`module add /${BUILDER_MODULE}.json`); + await mesh(`assign anchor ${BUILDER_MODULE}`); + await composeTheBusUsers(); + await mesh("push anchor", 600_000); + const by = Date.now() + 180_000; + while (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) { + if (Date.now() > by) { + assert.fail(`the mesh never delivered the builder's bus credential to ${BUILDER_STATE}/broker:\n` + + (await on("anchor", "tail -30 /var/log/mesh-host.log")).out); + } + await new Promise((r) => setTimeout(r, 3000)); + } + } await must("anchor", `mkdir -p /var/lib/mesh-builder`); await must("anchor", `pgrep -x mesh-builder >/dev/null || ` + - `(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` + + `(MESH_BROKER_FILE=${BUILDER_STATE}/broker MESH_NODE=anchor MESH_REGISTRY=${registry} ` + `MESH_WORKSPACE=/var/lib/mesh-builder ` + `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`); + // Holding the seat, not merely started: a builder refused by the bus exits, and a build asked of a + // seat nobody holds waits out its whole timeout before saying anything. + const running = await on("anchor", `pgrep -x mesh-builder >/dev/null`); + assert.ok(running.ok, `the builder did not stay up:\n${(await on("anchor", "tail -30 /var/log/mesh-builder.log")).out}`); } before(async () => { @@ -264,8 +309,7 @@ before(async () => { const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`); assert.equal(running.out.trim(), "yes", "the host did not come back after a restore, so nothing would apply anything"); - // The hand-started builder is memory too, and the snapshot is disk. - if (builder) await startBuilder(); + // The hand-started builder is memory too; the build test starts it again where it is needed. console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` + `and started the host again`); @@ -290,9 +334,8 @@ before(async () => { await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`); - // A build machine, so anything here can ask the mesh to build something. Placed rather than - // assumed: nothing else in this scenario would start one. - if (builder) await startBuilder(); + // No build machine yet: the builder holds its seat on a credential the mesh delivers, so it is + // started by the test that builds, once the anchor has joined (startBuilder). if (warming) { // Snapshotted only now, with everything up: a state worth returning to is the one after the // part nobody wants to repeat. @@ -551,11 +594,11 @@ test("a declaration waits for a machine that is switched off", { skip, timeout: // Everything needed to tell "the message was never queued" from "the host never read it". const log = await on("laptop", `tail -20 /var/log/mesh-host.log`); const queues = await on("anchor", - `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`); + `curl -s 'http://127.0.0.1:8222/jsz?consumers=true' 2>&1 | head -c 4000`); const owned = await on("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`); assert.fail(`a declaration sent to a switched-off machine was lost\n` + - `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` + + `--- the host's log ---\n${log.out}\n--- the bus's streams and consumers ---\n${queues.out}\n` + `--- what each machine last did ---\n${owned.out}`); } assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited"); @@ -1199,9 +1242,8 @@ test("a new commit reaches a machine that is already running the old one", { // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the // question "did my change go out?". This is that question, end to end — a commit, a build, a // catalogue, and a machine that ends up running what the source says. - // The hand-started builder does not outlive a broker restart, and the foundation's broker is - // recreated when the first push reconciles it: a builder is (re)started here, where a build is - // asked for. The mesh's own builder is a module with a restart policy and needs none of this. + // The builder needs the anchor joined — its credential is the mesh's to deliver — so it is started + // here, where a build is asked for. The mesh's own build agent is a module with a restart policy. await startBuilder(); const repo = "/var/lib/mesh/builder/repositories/delivered"; const write = async (what: string) =>