A machine in the mesh builds a module, and the catalogue records it

The chain this closes: a repository exists, the mesh asks for it, a build
machine takes the work, publishes what it made, and the catalogue then
says what the module is, which commit it came from, and — after the
source moves — that it is behind.

Three assertions, against a real broker and registry, because what is
under test is four processes agreeing over a wire:

- the mesh asks, a machine builds, and the artifact is really in the
  registry at the digest the manifest names
- a build that cannot succeed says why and records nothing. A failure
  that is silent is indistinguishable from a builder that is not running
- the source moving makes the catalogue say "behind", and rebuilding
  catches it up

git is now in the base image, with the same reasoning as docker and
wireguard-tools: a machine that builds modules clones them, and a sealed
scenario cannot install anything. Read back from `git --version` rather
than from the package manager — an installed package is not a
capability, and a build machine whose clone fails does so three minutes
into a scenario with the failure reported as a build problem rather than
a lab one.

src/lifecycle/base.ts

@@ -65,6 +65,12 @@ export async function buildBaseImage(
     log("  installing the tools for the private network");
     await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "wireguard-tools"], 600_000);
 
+    // And git, for the same reason again: a machine that builds modules clones them, and a sealed
+    // scenario cannot install it. On a real build machine the mesh installs it as a package like
+    // anything else — the lab is the special case, because its machines reach no mirror.
+    log("  installing git, so a machine can build modules");
+    await incus(["exec", BUILDER, "--", "pacman", "-S", "--noconfirm", "git"], 600_000);
+
     // Trust the documentation ranges as plain-HTTP registries.
     //
     // A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime
@@ -90,6 +96,18 @@ export async function buildBaseImage(
     }
     log(`  ${wg.trim()}`);
 
+    // The same, for git. An installed package is not a capability, and this is the one place to
+    // catch it — a build machine whose clone fails does so three minutes into a scenario, with
+    // the failure reported as a build problem rather than a lab one.
+    const git = await incusOk(["exec", BUILDER, "--", "git", "--version"], 60_000);
+    if (!git?.trim()) {
+      throw new BaseImageError(
+        `git was installed in ${BUILDER} and \`git --version\` does not answer. Publishing ` +
+          `this would give every scenario a machine that cannot build a module.`,
+      );
+    }
+    log(`  ${git.trim()}`);
+
     // Read back from the runtime, not from the package manager. An installed package is not a
     // capability (novox/hq 04-ISSUES/007), and this is the one place to catch that — after
     // publishing, every scenario pays for it instead.

test/integration/builds.test.ts

@@ -0,0 +1,168 @@
+/**
+ * A machine in the mesh builds a module, and the mesh records what came out.
+ *
+ * The chain this closes: a repository exists, the mesh asks for it to be built, a build machine
+ * takes the work, publishes what it made, and the catalogue then says what the module is, which
+ * commit it came from, and — after the source moves — that it is behind.
+ *
+ * Against a real broker and a real registry, because what is under test is that four processes
+ * agree over a wire. Everything either side of the wire is already asserted in its own suite.
+ *
+ *   MESH_LAB_HOST_BINARY   a built mesh-host
+ *   MESH_LAB_BUNDLE        the substrate bundle
+ *   MESH_LAB_BUILDER       a built mesh-builder
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, readFileSync } from "node:fs";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll } from "./harness.ts";
+import { incus } from "../../src/incus/client.ts";
+import { machineName } from "../../src/lifecycle/names.ts";
+
+const capability = await labIsUsable();
+const host = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+const builder = process.env["MESH_LAB_BUILDER"] ?? "";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !host || !existsSync(host)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a substrate bundle"
+      : !builder || !existsSync(builder)
+        ? "MESH_LAB_BUILDER is not set to a built mesh-builder"
+        : false;
+
+const SCENARIO = "first-node";
+const MACHINE = "anchor";
+let instanceId = "";
+let registry = "";
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(command: string): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, MACHINE, [
+    "sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
+  ]);
+  const marker = stdout.lastIndexOf("__exit=");
+  return { out: stdout.slice(0, marker), ok: Number(stdout.slice(marker + 7).trim()) === 0 };
+}
+
+async function must(command: string): Promise<string> {
+  const { out, ok } = await on(command);
+  if (!ok) throw new Error(`${command}\n${out}`);
+  return out;
+}
+
+async function mesh(command: string): Promise<string> {
+  return must(`docker exec mesh-control /mesh-control ${command}`);
+}
+
+/** The bundle, pointed at this scenario's own registry. */
+function bundleFor(images: string[]): string {
+  let text = readFileSync(bundle, "utf8");
+  for (const pinned of images) {
+    const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
+    const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
+    text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), pinned);
+  }
+  return text;
+}
+
+before(async () => {
+  if (skip) return;
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
+  instanceId = raised.instanceId;
+  const first = raised.images[0];
+  assert.ok(first, "the scenario stocked no images, so there is no registry to publish to");
+  registry = first.slice(0, first.indexOf("/"));
+
+  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
+  await must(`${HOST_PATH} apply /tmp/substrate.lock`);
+
+  // A module repository on the machine. Local rather than fetched, because what is under test is
+  // the mesh's chain and not whether the lab can reach a forge.
+  await must(`mkdir -p /root/shell/files`);
+  await must(`printf %s ${quote(JSON.stringify({
+    module: "shell",
+    version: "1",
+    provides: ["login-shell"],
+    build: { artifacts: [{ name: "config", kind: "archive", from: "files" }] },
+    resources: [
+      { id: "package", type: "package", package: "zsh" },
+      { id: "operator", type: "user", name: "operator", shell: "/bin/sh" },
+      {
+        id: "dotfiles", type: "archive", artifact: "config",
+        path: "/home/operator/.config/shell", owner: "operator",
+      },
+    ],
+  }))} > /root/shell/module.json`);
+  await must(`printf %s "alias ll='ls -l'\n" > /root/shell/files/aliases.zsh`);
+  await must(`cd /root/shell && git init -q . && git add -A && ` +
+    `git -c user.email=lab -c user.name=lab commit -qm first`);
+
+  await incus([
+    "file", "push", builder, `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-builder`,
+    "--mode", "0755",
+  ], 180_000);
+  // The build machine, holding its own broker credential and nothing else.
+  await must(
+    `MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
+      `MESH_WORKSPACE=/var/lib/mesh-builder ` +
+      `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`,
+  );
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("the mesh asks, a machine builds, and the catalogue records it", { skip, timeout: 900_000 }, async () => {
+  const said = await mesh("build /root/shell --wait 300s");
+  assert.match(said, /built on/, said);
+  assert.match(said, /config\s+archive/, `nothing was published:\n${said}`);
+
+  const listed = await mesh("module list");
+  assert.match(listed, /^shell\s+1\s+built [0-9a-f]{8}/m, listed);
+
+  // And the artifact is really there, at the digest the manifest names.
+  const digest = /blobs\/(sha256:[0-9a-f]{64})/.exec(said);
+  assert.ok(digest, `the build named no digest:\n${said}`);
+  const head = await on(`curl -sfI ${registry}/v2/shell/config/blobs/${digest[1]} >/dev/null`);
+  assert.ok(head.ok, "the registry does not have the blob the manifest points at");
+});
+
+test("a build that cannot succeed says why, and records nothing", { skip, timeout: 600_000 }, async () => {
+  // A failure is a result. A build that fails silently is indistinguishable from a builder that
+  // is not running, and those want completely different responses.
+  const { out, ok } = await on(
+    `docker exec mesh-control /mesh-control build /root/does-not-exist --wait 120s`,
+  );
+  assert.equal(ok, false, "a build of nothing reported success");
+  assert.match(out, /could not build/, out);
+  const listed = await mesh("module list");
+  assert.doesNotMatch(listed, /does-not-exist/, "a failed build was recorded");
+});
+
+test("when the source moves, the catalogue says the module is behind", { skip, timeout: 600_000 }, async () => {
+  await must(`cd /root/shell && printf %s "alias la='ls -la'\n" >> files/aliases.zsh && ` +
+    `git add -A && git -c user.email=lab -c user.name=lab commit -qm second`);
+  const moved = (await must(`cd /root/shell && git rev-parse HEAD`)).trim();
+
+  await mesh(`module moved shell ${moved}`);
+  assert.match(await mesh("module list"), /^shell\s+1\s+behind [0-9a-f]{8} < [0-9a-f]{8}/m);
+
+  // And building again catches it up, with a different digest because the content differs.
+  const rebuilt = await mesh("build /root/shell --wait 300s");
+  assert.match(rebuilt, new RegExp(`built on .* from ${moved.slice(0, 8)}`), rebuilt);
+  assert.match(await mesh("module list"), /^shell\s+1\s+built [0-9a-f]{8}/m);
+});