mesh-lab/test/integration/builds.test.ts

/**
 * A machine in the mesh builds a module, and the mesh records what came out.
 *
 * The chain this closes: a repository exists, the mesh asks for it to be built, a build machine
 * takes the work, publishes what it made, and the catalogue then says what the module is, which
 * commit it came from, and — after the source moves — that it is behind.
 *
 * Against a real broker and a real registry, because what is under test is that four processes
 * agree over a wire. Everything either side of the wire is already asserted in its own suite.
 *
 *   MESH_LAB_HOST_BINARY   a built mesh-host
 *   MESH_LAB_BUNDLE        the substrate bundle
 *   MESH_LAB_BUILDER       a built mesh-builder
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";

const capability = await labIsUsable();
const host = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const builder = process.env["MESH_LAB_BUILDER"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !host || !existsSync(host)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle"
      : !builder || !existsSync(builder)
        ? "MESH_LAB_BUILDER is not set to a built mesh-builder"
        : false;

const SCENARIO = "first-node";
const MACHINE = "anchor";
let instanceId = "";

/**
 * Where a build publishes to.
 *
 * **The mesh has a registry, and this is that one.** `mesh-catalog/modules/registry` serves the
 * mesh's artifact store on port 5000; a build publishes into it. This test starts the same image
 * on the machine directly rather than assigning the module, because what is under test is the
 * build chain and not module delivery.
 *
 * It used to publish into the registry the LAB raised inside the scenario — scenery pretending to
 * be upstream, which is the thing this change removed. A registry the mesh runs and a registry the
 * lab runs are different claims, and only the first exists in production.
 */
const ARTIFACT_STORE =
  "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
const registry = "127.0.0.1:5000";

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(command: string): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
  ]);
  const marker = stdout.lastIndexOf("__exit=");
  return { out: stdout.slice(0, marker), ok: Number(stdout.slice(marker + 7).trim()) === 0 };
}

async function must(command: string): Promise<string> {
  const { out, ok } = await on(command);
  if (!ok) throw new Error(`${command}\n${out}`);
  return out;
}

async function mesh(command: string): Promise<string> {
  return must(`docker exec mesh-control /mesh-control ${command}`);
}

/** The bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
  return substrateBundle(bundle, images);
}

before(async () => {
  if (skip) return;
  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
  instanceId = raised.instanceId;

  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
  await must(`${HOST_PATH} apply /tmp/substrate.lock`);

  // The mesh's artifact store, standing where the `registry` module would. Read back rather than
  // assumed: a builder publishing into a registry that never came up fails several minutes later,
  // as a manifest naming a blob nobody has.
  await must(
    `docker run -d --name mesh-registry --restart unless-stopped ` +
      `-p ${registry}:5000 ${ARTIFACT_STORE}`,
  );
  let serving = false;
  for (let i = 0; i < 30 && !serving; i++) {
    ({ ok: serving } = await on(`curl -sf http://${registry}/v2/ >/dev/null`));
    if (!serving) await new Promise((r) => setTimeout(r, 2_000));
  }
  assert.ok(serving, "the mesh's artifact store never answered, so a build has nowhere to publish");

  // A module repository on the machine. Local rather than fetched, because what is under test is
  // the mesh's chain and not whether the lab can reach a forge.
  await must(`mkdir -p /root/shell/files`);
  await must(`printf %s ${quote(JSON.stringify({
    module: "shell",
    version: "1",
    provides: ["login-shell"],
    build: { artifacts: [{ name: "config", kind: "archive", from: "files" }] },
    resources: [
      { id: "package", type: "package", package: "zsh" },
      { id: "operator", type: "user", name: "operator", shell: "/bin/sh" },
      {
        id: "dotfiles", type: "archive", artifact: "config",
        path: "/home/operator/.config/shell", owner: "operator",
      },
    ],
  }))} > /root/shell/module.json`);
  await must(`printf %s "alias ll='ls -l'\n" > /root/shell/files/aliases.zsh`);
  await must(`cd /root/shell && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm first`);

  await incus([
    "file", "push", builder, `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-builder`,
    "--mode", "0755",
  ], 180_000);
  // The build machine, holding its own broker credential and nothing else.
  await must(
    `MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
      `MESH_WORKSPACE=/var/lib/mesh-builder ` +
      `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`,
  );
}, { timeout: 1_800_000 });

after(async () => {
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("the mesh asks, a machine builds, and the catalogue records it", { skip, timeout: 900_000 }, async () => {
  const said = await mesh("build /root/shell --wait 300s");
  assert.match(said, /built on/, said);
  assert.match(said, /config\s+archive/, `nothing was published:\n${said}`);

  const listed = await mesh("module list");
  assert.match(listed, /^shell\s+1\s+built [0-9a-f]{8}/m, listed);

  // And the artifact is really there, at the digest the manifest names.
  const digest = /blobs\/(sha256:[0-9a-f]{64})/.exec(said);
  assert.ok(digest, `the build named no digest:\n${said}`);
  const head = await on(`curl -sfI ${registry}/v2/shell/config/blobs/${digest[1]} >/dev/null`);
  assert.ok(head.ok, "the registry does not have the blob the manifest points at");
});

test("a build that cannot succeed says why, and records nothing", { skip, timeout: 600_000 }, async () => {
  // A failure is a result. A build that fails silently is indistinguishable from a builder that
  // is not running, and those want completely different responses.
  const { out, ok } = await on(
    `docker exec mesh-control /mesh-control build /root/does-not-exist --wait 120s`,
  );
  assert.equal(ok, false, "a build of nothing reported success");
  assert.match(out, /could not build/, out);
  const listed = await mesh("module list");
  assert.doesNotMatch(listed, /does-not-exist/, "a failed build was recorded");
});

test("when the source moves, the catalogue says the module is behind", { skip, timeout: 600_000 }, async () => {
  await must(`cd /root/shell && printf %s "alias la='ls -la'\n" >> files/aliases.zsh && ` +
    `git add -A && git -c user.email=lab -c user.name=lab commit -qm second`);
  const moved = (await must(`cd /root/shell && git rev-parse HEAD`)).trim();

  await mesh(`module moved shell ${moved}`);
  assert.match(await mesh("module list"), /^shell\s+1\s+behind [0-9a-f]{8} < [0-9a-f]{8}/m);

  // And building again catches it up, with a different digest because the content differs.
  const rebuilt = await mesh("build /root/shell --wait 300s");
  assert.match(rebuilt, new RegExp(`built on .* from ${moved.slice(0, 8)}`), rebuilt);
  assert.match(await mesh("module list"), /^shell\s+1\s+built [0-9a-f]{8}/m);
});