The lab runs on the laptop again: the walk's builder on the bus, runtime images on node-tools, enumeration tests independent of the host's incus (hq issues 307, 308) #63
@@ -198,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a
|
||||
says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
|
||||
scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
|
||||
source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
|
||||
beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
|
||||
where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
|
||||
beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing,
|
||||
or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`,
|
||||
built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the
|
||||
launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs
|
||||
`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a
|
||||
`MESH_BROKER_URL`, never the served module's own: the runtime refuses that.
|
||||
`--no-build` skips this too, and then the bed runs whatever image the store holds.
|
||||
|
||||
Check before running a long suite — it says which of these are missing rather than skipping
|
||||
|
||||
@@ -114,6 +114,25 @@ var Register = []Replay{
|
||||
Asserts: "a merge adding a module asks the build seat for it, at the branch merged into, and opens no plan",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay300", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
||||
// A drill counted as a repair (2026-10-07): S15 read two deliberate drills, recorded through hand-act
|
||||
// record for want of a verb, as a cause repaired twice. In a file of its own, since replays_test.go holds
|
||||
// replays of later commits that do not build at the commit before this fix. Home is the controller's
|
||||
// commit that added it; it stays reachable once that pull request merges.
|
||||
{ID: "R292", Issue: 292, Kind: InRepository, Repository: "mesh-controller", Fix: "863ebd4",
|
||||
What: "two drills of ADR 0240, each with the operator's word, recorded through hand-act record --cause drill, " +
|
||||
"raised mesh.hand-acts.drill.healer-wanted",
|
||||
Asserts: "a drill recorded in the hand-act log wants no healer, however it was recorded",
|
||||
Package: "./cmd/mesh-controller", Test: "TestReplay292", Files: []string{"cmd/mesh-controller/replay292_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// A seat's new verb deadlocked across two repositories (2026-10-07): the controller promising checks
|
||||
// refused the delivery seat's holder that did not serve it, and a holder serving it was refused by the
|
||||
// controller that did not promise it.
|
||||
{ID: "R298", Issue: 298, Kind: InRepository, Repository: "mesh-controller", Fix: "f6aea4b",
|
||||
What: "the delivery seat's new verb checks could land in neither repository first: each side's claim " +
|
||||
"check refused the other's holder",
|
||||
Asserts: "the delivery seat's holder holds it both before and after it serves checks",
|
||||
Package: "./internal/catalogue", Test: "TestReplay298", Files: []string{"internal/catalogue/replays_test.go"},
|
||||
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
||||
// The push a recorded build waits for, counted as a repair (2026-10-07): a test in the controller, in
|
||||
// the fix's own commit, recording two such pushes as the seat's push records them.
|
||||
{ID: "R301", Issue: 301, Kind: InRepository, Repository: "mesh-controller", Fix: "e92a3fe",
|
||||
|
||||
@@ -30,9 +30,9 @@ machines:
|
||||
memory: 2GiB
|
||||
|
||||
images:
|
||||
# The packet filter's seat runtime (novox/hq ADR 0170): the filter module now serves its verbs from
|
||||
# a runtime the mesh builds, so a bed that installs the filter stocks it.
|
||||
- mesh-runtime-nftables:development
|
||||
# No packet-filter runtime: the filter module's verbs are a tools bundle the node's runtime launches
|
||||
# (novox/hq ADR 0170, 0193), not a container of its own, and what this walk needs of the module is
|
||||
# the ruleset it loads — which its resources do. Stocking one built an image nothing ran.
|
||||
- mesh-controller:development
|
||||
# And the builder, because it is a module the mesh assigns rather than a program somebody
|
||||
# starts by hand — which is the only way its credential can be one the mesh delivered.
|
||||
|
||||
+115
-57
@@ -1,10 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's
|
||||
# compiled code, which serves that module's tools and runs its events/provisioner under the module's
|
||||
# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module.
|
||||
# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
|
||||
# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
|
||||
#
|
||||
# build-module-runtime.sh <module> <output.tar>
|
||||
# -> tags mesh-runtime-<module>:development and saves it to <output.tar>
|
||||
#
|
||||
# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
|
||||
# imported every module's compiled code into one process, and this script copied its dist and its
|
||||
# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
|
||||
# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
|
||||
# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
|
||||
# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
|
||||
# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
|
||||
# so the old script failed at its first step.
|
||||
#
|
||||
# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
|
||||
# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
|
||||
# entrypoint lands where it is named);
|
||||
# 2. writes each entrypoint's launcher, executable;
|
||||
# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
|
||||
# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
|
||||
# module's tools, events and provisioner entrypoints, whichever exist.
|
||||
#
|
||||
# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
|
||||
# and is not repeated here.
|
||||
#
|
||||
# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
|
||||
# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
|
||||
# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
|
||||
# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
|
||||
# module's own.
|
||||
#
|
||||
# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
|
||||
# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
|
||||
set -euo pipefail
|
||||
|
||||
MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
|
||||
@@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
|
||||
MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
|
||||
MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
|
||||
MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
|
||||
MOD="$MESH_CATALOG/modules/$MODULE"
|
||||
# Absolute, because the steps below run from inside the module.
|
||||
for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
|
||||
[ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
|
||||
printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
|
||||
done
|
||||
# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
|
||||
if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
|
||||
MOD="$MESH_CATALOG/$MODULE"
|
||||
TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
|
||||
BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
|
||||
RUNTIME_SRC="$MESH_TOOLS/node-tools"
|
||||
[ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
|
||||
[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
|
||||
command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
|
||||
|
||||
# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
|
||||
[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
|
||||
( cd "$MESH_SDK" && npm run build >/dev/null )
|
||||
( cd "$MESH_TOOLS" && npm run build >/dev/null )
|
||||
# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools,
|
||||
# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints
|
||||
# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage
|
||||
# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls
|
||||
# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient
|
||||
# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the
|
||||
# ambient declaration is in the program even though the dep is only installed into the image below.
|
||||
SRCS=(); for f in \
|
||||
client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
|
||||
adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \
|
||||
pg.d.ts; do
|
||||
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
||||
|
||||
# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
|
||||
# builder reads them; a module declaring none falls back to the ones the runtime serves.
|
||||
ENTRIES_JS="$(node -e '
|
||||
const m = require(process.argv[1]);
|
||||
const out = new Set();
|
||||
for (const a of (m.build && m.build.artifacts) || [])
|
||||
if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
|
||||
process.stdout.write([...out].join(" "));
|
||||
' "$MOD/module.json")"
|
||||
if [ -z "$ENTRIES_JS" ]; then
|
||||
for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
|
||||
done
|
||||
fi
|
||||
SRCS=(); for e in $ENTRIES_JS; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
|
||||
SRCS+=("${e%.js}.ts")
|
||||
done
|
||||
[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
|
||||
# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
|
||||
# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
|
||||
for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
|
||||
|
||||
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
||||
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
||||
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
|
||||
# the types; the image takes the SDK from MESH_SDK below.
|
||||
# as a package copy from the sibling checkout (never a link) when absent.
|
||||
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
||||
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
||||
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
||||
fi
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
|
||||
|
||||
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
||||
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
||||
cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules"
|
||||
# And the sdk, from the sibling this script just built, whatever form the installed tree holds it
|
||||
# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised —
|
||||
# true only on a workstation where somebody had linked them, and false the moment the runtime's
|
||||
# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing
|
||||
# compiled in it. The image built then looked fine and every entry point inside it pointed at
|
||||
# nothing.
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
mkdir -p "$STAGE/node_modules/@novox"
|
||||
cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules"
|
||||
# **The image runs compiled code and never compiles any**, so it does not need a compiler. The
|
||||
# tree copied above is the runtime's full install, development dependencies and all — and the
|
||||
# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing:
|
||||
# tsc runs on the workstation a few lines above, not in here.
|
||||
#
|
||||
# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one
|
||||
# of them a git URL with no registry behind it — and could drop something the image needs.
|
||||
rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types"
|
||||
mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist"
|
||||
cp "$MESH_TOOLS/package.json" "$STAGE/package.json"
|
||||
DIST="$STAGE/modules/$MODULE/dist"
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
|
||||
# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"
|
||||
( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
|
||||
--rootDir . --outDir "$DIST" 1>&2 )
|
||||
# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
|
||||
printf '{"type":"module","private":true}\n' > "$DIST/package.json"
|
||||
|
||||
# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
|
||||
# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
|
||||
for e in $ENTRIES_JS; do
|
||||
[ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
|
||||
launcher="$DIST/${e%.js}.serve.mjs"
|
||||
cat > "$launcher" <<LAUNCHER
|
||||
#!/usr/bin/env node
|
||||
// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
|
||||
// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
|
||||
import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
|
||||
await import("./$(basename "$e")");
|
||||
await serveRegisteredOverStdio();
|
||||
LAUNCHER
|
||||
chmod 0755 "$launcher"
|
||||
done
|
||||
|
||||
# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
|
||||
# runtime dependencies of its own.
|
||||
mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
|
||||
cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
|
||||
|
||||
# A module may declare its own third-party runtime deps (the anthropic-manager seals with
|
||||
# tweetnacl-sealedbox-js). The shared node_modules copied above carries the common packages and
|
||||
# @novox/* — but not a module's private deps. Install those under the module itself, so Node
|
||||
# resolves them from /app/modules/<module>/node_modules and still falls back to the shared tree
|
||||
# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a
|
||||
# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.)
|
||||
MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')"
|
||||
# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
|
||||
# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
|
||||
# excluded: there is no public registry for them, and the SDK is staged above.
|
||||
MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
|
||||
if [ -n "$MOD_DEPS" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
|
||||
fi
|
||||
|
||||
# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist.
|
||||
ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e"
|
||||
# The runtime itself: node-tools, static, from mesh-tools' Go module.
|
||||
( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
|
||||
|
||||
# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
|
||||
# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
|
||||
# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
|
||||
# every entrypoint.
|
||||
SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
|
||||
done
|
||||
|
||||
# A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
|
||||
@@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" <<DOCKER
|
||||
FROM $BASE
|
||||
WORKDIR /app
|
||||
$EXTRA
|
||||
COPY package.json ./
|
||||
COPY node-tools /usr/local/bin/node-tools
|
||||
COPY node_modules ./node_modules
|
||||
COPY dist ./dist
|
||||
COPY modules ./modules
|
||||
ENV MESH_TOOL_MODULES=$ENTRIES
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
ENV MESH_TOOL_MODULES=$SERVED
|
||||
ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
|
||||
DOCKER
|
||||
docker build -t "$TAG" "$STAGE"
|
||||
docker save -o "$OUT" "$TAG"
|
||||
echo "built $TAG (entrypoints: $ENTRIES) -> $OUT"
|
||||
echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"
|
||||
|
||||
+18
-1
@@ -20,7 +20,24 @@ import { around, log, shorten } from "../log.ts";
|
||||
* predates the group grant cannot reach it — which is a real thing that happens on the
|
||||
* machine that just installed it.
|
||||
*/
|
||||
const INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
let INCUS = (process.env["MESH_LAB_INCUS"] ?? "incus").split(" ").filter(Boolean);
|
||||
|
||||
/**
|
||||
* Point every call at another program, and say what it was pointed at before.
|
||||
*
|
||||
* **For a test about how the lab reads a failure, never about the hypervisor.** The command was
|
||||
* read from `MESH_LAB_INCUS` once, when this module loaded — and a test file setting that variable
|
||||
* in its body set it too late, because an ES module's imports are evaluated before its body. So the
|
||||
* enumeration tests asked the real incus wherever one was installed and reachable: they passed on a
|
||||
* machine without incus (the spawn fails) and failed on the workstation that runs the lab, where
|
||||
* listing succeeds. Injected here, a test names the failing program and gets it, on every machine.
|
||||
*/
|
||||
export function useIncusCommand(argv: string[]): string[] {
|
||||
if (argv.length === 0 || !argv[0]) throw new Error("an incus command needs a program to run");
|
||||
const was = INCUS;
|
||||
INCUS = [...argv];
|
||||
return was;
|
||||
}
|
||||
|
||||
export interface IncusResult {
|
||||
stdout: string;
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
// Set before importing: the client reads MESH_LAB_INCUS once, at module load.
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
process.env["MESH_LAB_INCUS"] = "false";
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { instanceExists, taggedInstances, taggedNetworks } from "../src/incus/client.ts";
|
||||
import { instanceExists, taggedInstances, taggedNetworks, useIncusCommand } from "../src/incus/client.ts";
|
||||
|
||||
// `false` is a real program that exits non-zero and prints nothing — which is also the worst
|
||||
// case, because an empty stderr is how a failure arrives with no explanation.
|
||||
//
|
||||
// **Injected, never set in the environment here.** This file used to set MESH_LAB_INCUS in its body,
|
||||
// which runs after the imports above — so the client had already read the variable, and these tests
|
||||
// asked the real incus: green on a machine without one, red on the workstation that runs the lab.
|
||||
useIncusCommand(["false"]);
|
||||
|
||||
/**
|
||||
* "I cannot see" must never be answered as "there is nothing there."
|
||||
|
||||
@@ -376,9 +376,10 @@ function shellQuote(s: string): string {
|
||||
*/
|
||||
export async function deriveTheFilterOn(o: {
|
||||
machine: string; node: string; hubPort: number;
|
||||
/** The images the machines hold. Given, the filter module's runtime — the packet-filter seat's,
|
||||
* which the mesh would build (novox/hq ADR 0170) — is the stocked one, as for any catalogue
|
||||
* module a bed installs; the scenario must then stock `mesh-runtime-nftables:development`. */
|
||||
/** The images the machines hold. Given, the manifest is the lab's form of the catalogue's
|
||||
* (catalogueModule): its build section gone, since the lab builds nothing. The filter's verbs are a
|
||||
* tools bundle the node's runtime launches (novox/hq ADR 0170, 0193), so no container of the
|
||||
* module names an image and none need be stocked; what a bed needs of it is the ruleset it loads. */
|
||||
held?: HeldImage[];
|
||||
must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
|
||||
mesh: (command: string, timeoutMs?: number) => Promise<string>;
|
||||
|
||||
@@ -214,8 +214,33 @@ function tokenFrom(said: string): string {
|
||||
return found;
|
||||
}
|
||||
|
||||
/** The builder started by hand on the anchor, against the foundation broker's plain port on
|
||||
* loopback — the one that builds until a builder module can (see the retired test's note). */
|
||||
/**
|
||||
* The build machine's module: a holder of the node-build-agent seat, with the bus credential the mesh
|
||||
* seals to the machine and nothing else.
|
||||
*
|
||||
* **On the bus, the way today's build agent is** (novox/hq ADR 0106, 0127, 0131). The builder used to
|
||||
* be started against the old broker's guest account on loopback; the mesh moved to NATS, the builder
|
||||
* dials only a credential the mesh delivered (`MESH_BROKER_FILE`), and that account exists only as a
|
||||
* module's, composed into the bus's user list. The catalogue's `build-agent` is that module, but it
|
||||
* runs the builder as a container from an artifact the mesh builds and needs the artifact store and
|
||||
* package registry bound; this bed builds nothing before it has a builder, so it declares the same
|
||||
* claim and the same sealed credential and starts the binary it was pointed at itself.
|
||||
*/
|
||||
const BUILDER_MODULE = "lab-builder";
|
||||
const BUILDER_STATE = `/var/lib/mesh/${BUILDER_MODULE}`;
|
||||
const BUILDER_MANIFEST = JSON.stringify({
|
||||
module: BUILDER_MODULE, version: "1",
|
||||
claims: [{ name: "node-build-agent", scope: "node", serves: ["current", "kill", "pause", "resume"] }],
|
||||
"own-secrets": { broker: `${BUILDER_STATE}/broker` },
|
||||
resources: [{ id: "mesh-state", type: "directory", path: BUILDER_STATE, mode: "0700" }],
|
||||
});
|
||||
|
||||
/**
|
||||
* The builder, on the anchor, holding the build seat over the bus with the credential the mesh sealed
|
||||
* to the machine. Assigning the module issues the account (novox/hq issue 203); the bus here is the
|
||||
* bundle's, so its user list is placed by hand (composeTheBusUsers); the push writes the credential
|
||||
* where the module declared it. Idempotent: a warm restore or a second call starts only what is gone.
|
||||
*/
|
||||
async function startBuilder(): Promise<void> {
|
||||
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
|
||||
// return, so it is pushed whenever the machine has none.
|
||||
@@ -225,11 +250,31 @@ async function startBuilder(): Promise<void> {
|
||||
"--mode", "0755",
|
||||
], 180_000);
|
||||
}
|
||||
if (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
await must("anchor", `printf %s ${quote(BUILDER_MANIFEST)} > /tmp/${BUILDER_MODULE}.json && ` +
|
||||
`docker cp /tmp/${BUILDER_MODULE}.json mesh-controller:/${BUILDER_MODULE}.json`);
|
||||
await mesh(`module add /${BUILDER_MODULE}.json`);
|
||||
await mesh(`assign anchor ${BUILDER_MODULE}`);
|
||||
await composeTheBusUsers();
|
||||
await mesh("push anchor", 600_000);
|
||||
const by = Date.now() + 180_000;
|
||||
while (!(await on("anchor", `test -s ${BUILDER_STATE}/broker`)).ok) {
|
||||
if (Date.now() > by) {
|
||||
assert.fail(`the mesh never delivered the builder's bus credential to ${BUILDER_STATE}/broker:\n` +
|
||||
(await on("anchor", "tail -30 /var/log/mesh-host.log")).out);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
||||
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
|
||||
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
||||
`(MESH_BROKER_FILE=${BUILDER_STATE}/broker MESH_NODE=anchor MESH_REGISTRY=${registry} ` +
|
||||
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
||||
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
|
||||
// Holding the seat, not merely started: a builder refused by the bus exits, and a build asked of a
|
||||
// seat nobody holds waits out its whole timeout before saying anything.
|
||||
const running = await on("anchor", `pgrep -x mesh-builder >/dev/null`);
|
||||
assert.ok(running.ok, `the builder did not stay up:\n${(await on("anchor", "tail -30 /var/log/mesh-builder.log")).out}`);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
@@ -264,8 +309,7 @@ before(async () => {
|
||||
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
||||
assert.equal(running.out.trim(), "yes",
|
||||
"the host did not come back after a restore, so nothing would apply anything");
|
||||
// The hand-started builder is memory too, and the snapshot is disk.
|
||||
if (builder) await startBuilder();
|
||||
// The hand-started builder is memory too; the build test starts it again where it is needed.
|
||||
|
||||
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
||||
`and started the host again`);
|
||||
@@ -290,9 +334,8 @@ before(async () => {
|
||||
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);
|
||||
|
||||
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
||||
// assumed: nothing else in this scenario would start one.
|
||||
if (builder) await startBuilder();
|
||||
// No build machine yet: the builder holds its seat on a credential the mesh delivers, so it is
|
||||
// started by the test that builds, once the anchor has joined (startBuilder).
|
||||
if (warming) {
|
||||
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
||||
// part nobody wants to repeat.
|
||||
@@ -551,11 +594,11 @@ test("a declaration waits for a machine that is switched off", { skip, timeout:
|
||||
// Everything needed to tell "the message was never queued" from "the host never read it".
|
||||
const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
|
||||
const queues = await on("anchor",
|
||||
`docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
|
||||
`curl -s 'http://127.0.0.1:8222/jsz?consumers=true' 2>&1 | head -c 4000`);
|
||||
const owned = await on("anchor",
|
||||
`docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
|
||||
assert.fail(`a declaration sent to a switched-off machine was lost\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
|
||||
`--- the host's log ---\n${log.out}\n--- the bus's streams and consumers ---\n${queues.out}\n` +
|
||||
`--- what each machine last did ---\n${owned.out}`);
|
||||
}
|
||||
assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
|
||||
@@ -1199,9 +1242,8 @@ test("a new commit reaches a machine that is already running the old one", {
|
||||
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
||||
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
||||
// catalogue, and a machine that ends up running what the source says.
|
||||
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
|
||||
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
|
||||
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
|
||||
// The builder needs the anchor joined — its credential is the mesh's to deliver — so it is started
|
||||
// here, where a build is asked for. The mesh's own build agent is a module with a restart policy.
|
||||
await startBuilder();
|
||||
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
||||
const write = async (what: string) =>
|
||||
|
||||
Reference in New Issue
Block a user