mesh-lab/test/integration/canary.test.ts

/**
 * The shortest run that would have caught today's faults.
 *
 * **A suite that takes forty minutes is a suite you find out from once a day.** Every fault found
 * on 2026-09-01 — a module pinned to an image that does not exist, a consumer given a password and
 * no name to present with it, a credential file nothing could read, a search for a password that
 * read the password as an option — would have shown up in the first three minutes of it. The other
 * thirty-seven proved things that were already working.
 *
 * So this runs first, on one machine, with the three images the mesh needs for itself and nothing
 * else. If it fails there is no point spending the rest.
 *
 * It is deliberately *not* a smaller copy of the full suite. It walks one path end to end — a mesh
 * comes up, a module reaches a machine, and a consumer gets a credential it can actually use —
 * because that path is where everything went wrong, and a canary that checks many things shallowly
 * is a canary nobody can read the failure of.
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";

import { raise } from "../../src/lifecycle/raise.ts";
import { exec, destroy } from "../../src/lifecycle/operate.ts";
import { loadScenario } from "../../src/declaration/parse.ts";
import { labIsUsable } from "./harness.ts";
import { pinnedInto } from "../../src/pinning.ts";

const capability = await labIsUsable();
const host = process.env["MESH_LAB_HOST_BINARY"] ?? "";
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !host || !existsSync(host)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle"
      : false;

const SCENARIO = "first-node";
const MACHINE = "anchor";
const HOST_PATH = "/usr/local/bin/mesh-host";
let instanceId = "";

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  return { out: stdout.slice(0, marker), ok: Number(stdout.slice(marker + 7).trim()) === 0 };
}

async function must(command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(command, timeoutMs);
  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
  return out;
}

const mesh = (command: string, timeoutMs?: number) =>
  must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);

before(async () => {
  if (skip) return;
  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
  });
  instanceId = raised.instanceId;
  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${
    pinnedInto(readFileSync(bundle, "utf8"), raised.images)}\nMESHBUNDLE`);
  await must(`${HOST_PATH} apply /tmp/substrate.lock`, 300_000);

  // **And the machine joins.** Applying the bundle raises a control plane; it does not tell that
  // control plane a machine exists. Leaving this out is what the first run of this canary found,
  // in under three minutes: the mesh answered, said "0 machine(s)", and every assignment after it
  // failed with `no node of that name`.
  await mesh(`node add ${MACHINE}`);
  const said = await mesh(`token issue --node ${MACHINE}`);
  const token = said.split("\n").map((l) => l.trim())
    .find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(token, `no token in:\n${said}`);
  await must(`${HOST_PATH} enrol --token ${quote(token)}`);

  // The host has to be running for a push to reach it.
  await must(`pgrep -x mesh-host >/dev/null || ` +
    `(setsid nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 < /dev/null & sleep 3)`);
});

after(async () => {
  // The canary owns its scenario and takes it down. Left standing it would hold a machine for
  // the forty minutes of the run it exists to protect.
  if (instanceId) await destroy(instanceId).catch(() => {});
});

test("a mesh comes up and answers", { skip, timeout: 600_000 }, async () => {
  const said = await mesh("status");
  assert.match(said, /anchor/, `the mesh does not know the machine it is running on:\n${said}`);
});

// One module, no images, nothing to stock. What is under test is the chain — added, assigned,
// resolved, planned, pushed, applied, reported — not what is at the end of it.
test("a module reaches the machine", { skip, timeout: 600_000 }, async () => {
  await must(`printf %s ${quote(JSON.stringify({
    module: "canary", version: "1",
    resources: [
      { id: "state", type: "directory", path: "/var/lib/canary", mode: "0700" },
      { id: "note", type: "file", path: "/var/lib/canary/it-arrived", content: "yes\n", mode: "0644" },
    ],
  }))} > /tmp/canary.json`);
  await must(`docker cp /tmp/canary.json mesh-control:/canary.json`);
  await mesh("module add /canary.json");
  await mesh(`assign ${MACHINE} canary`);
  await mesh(`push ${MACHINE}`, 300_000);

  assert.equal((await must(`cat /var/lib/canary/it-arrived`)).trim(), "yes");
  assert.match(await must(`stat -c %a /var/lib/canary`), /^700/);
});

// **The half that broke all day.** A provider and a consumer on one machine: the mesh makes a
// credential, tells the provider who asked, and gives the consumer a file it can read — with the
// name to present, which it could not have known (novox/hq 04-ISSUES/021, 022, 023).
test("a consumer gets a credential it can use", { skip, timeout: 600_000 }, async () => {
  await must(`printf %s ${quote(JSON.stringify({
    module: "canary-store", version: "1",
    provides: [{ name: "canary-database", scope: "mesh" }],
    serves: { "canary-database": { port: 5432 } },
    receives: { "canary-database": "/var/lib/canary-store/asked.json" },
    grants: { "canary-database": "/var/lib/canary-store/grants" },
    resources: [
      { id: "state", type: "directory", path: "/var/lib/canary-store", mode: "0700" },
      { id: "grants", type: "directory", path: "/var/lib/canary-store/grants", mode: "0700" },
    ],
  }))} > /tmp/canary-store.json`);
  await must(`printf %s ${quote(JSON.stringify({
    module: "canary-app", version: "1",
    requires: ["canary-database"],
    contributes: { "canary-database": { name: "canaryapp" } },
    binds: { "canary-database": "/var/lib/canary-app/where.json" },
    secrets: { "canary-database": "/var/lib/canary-app/password" },
    resources: [
      { id: "state", type: "directory", path: "/var/lib/canary-app", mode: "0700" },
      {
        id: "env", type: "file", path: "/var/lib/canary-app/database.env", mode: "0600",
        content: "PGHOST=${bound:canary-database:at}\nPGPORT=${bound:canary-database:port}\n" +
          "PGUSER=${bound:canary-database:as}\nPGPASSWORD=${secret:canary-database}\n",
      },
    ],
  }))} > /tmp/canary-app.json`);
  for (const name of ["canary-store", "canary-app"]) {
    await must(`docker cp /tmp/${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
    await mesh(`assign ${MACHINE} ${name}`);
  }
  await mesh(`push ${MACHINE}`, 300_000);

  const password = (await must(`cat /var/lib/canary-app/password`)).trim();
  assert.ok(password.length >= 40, `the consumer's credential is ${password.length} characters`);

  // Every hole filled, and filled with the right thing.
  const env = await must(`cat /var/lib/canary-app/database.env`);
  assert.match(env, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${env}`);
  assert.match(env, /^PGUSER=mesh_[a-z0-9_]+_canary_app$/m,
    `the consumer was not told what name to present:\n${env}`);
  assert.doesNotMatch(env, /\$\{/, `a placeholder reached the machine as a value:\n${env}`);
  assert.ok(env.includes(`PGPASSWORD=${password}`),
    `the file holds a different password from the credential file:\n${env}`);

  // And the provider was told who asked, which is what makes the credential real.
  const asked = await must(`cat /var/lib/canary-store/asked.json`);
  assert.match(asked, /canary-app/, `the provider was not told who asked:\n${asked}`);
  assert.match(asked, /"as": "mesh_[a-z0-9_]+_canary_app"/,
    `the provider was not told what to call the login:\n${asked}`);
});