mesh-lab/test/integration/mesh.test.ts

/**
 * A mesh, raised from nothing, joined by two machines, delivering a credential neither the mesh
 * nor the broker can read.
 *
 * Everything before this proves a part. This proves the parts meet — which is the thing the
 * project keeps saying cannot be checked any other way (novox/hq ADR 0001: every fault of
 * 2026-08-22 was found in production because nothing could be stood up locally).
 *
 * It needs a host binary and the substrate bundle:
 *
 *   MESH_LAB_HOST_BINARY=.../mesh-host
 *   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
 *
 * The bundle's image references are rewritten to the ones this scenario's own registry serves.
 * A digest belongs to whatever registry serves it, so a committed bundle names a registry that is
 * not this one — rewriting is what makes it applicable rather than a placeholder to tidy away.
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";

/** Whether this run keeps its mesh for the next one. Off unless asked for. */
const warming = process.env["MESH_LAB_WARM"] === "1";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
/** mesh-control's `examples/modules`, so the manifests proven here are the ones that ship. */
const moduleExamples = process.env["MESH_LAB_MODULES"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
      : false;

const SCENARIO = "two-nodes";
let instanceId = "";
/** The scenario's own registry, which serves the images a module may mirror. */
let registry = "";
/** What that registry actually serves, by repository. */
let stocked: string[] = [];

/**
 * The pinned reference for one of the scenario's images.
 *
 * By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and
 * asking for it fails with "not found", which reads like a missing image rather than a naming
 * convention. A digest is also what a declaration pins, so this is the reference a module would
 * really carry.
 */
function pinned(repository: string): string {
  const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
  assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
  return found;
}

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  // Each part on its own line, and stderr redirected once for the whole script.
  //
  // It was `${command} 2>&1; echo ...` on a single line, which quietly broke every command
  // containing a heredoc: the terminator line became `MARKER 2>&1; echo ...`, matched nothing, and
  // the heredoc swallowed the rest of the script — including the echo. `exec 2>&1` needs no
  // trailing text on the command's last line, so a heredoc terminates where it says it does.
  const { stdout } = await exec(instanceId, machine, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) {
    // **Never success.** `Number("")` is 0, so a missing marker used to read as exit 0 — a
    // command whose output was swallowed reported that it worked, which is the one answer a test
    // harness must never give.
    return { out: stdout, ok: false };
  }
  const said = stdout.slice(marker + 7).trim();
  return { out: stdout.slice(0, marker), ok: said === "0" };
}

async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(machine, command, timeoutMs);
  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
  return out;
}

/** The control plane, which runs in a container on the first node. */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}

/**
 * The bundle, with every image reference pointed at this scenario's registry.
 *
 * Matched by repository rather than by the whole reference, because the address and the digest
 * both differ from whatever the committed bundle names — and a bundle that names the wrong
 * registry is not wrong, it is built for a different target.
 */
function bundleFor(images: string[]): string {
  let text = readFileSync(bundle, "utf8");
  for (const pinned of images) {
    const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
    const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
    text = text.replaceAll(
      new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"),
      pinned,
    );
  }
  return text;
}

/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

before(async () => {
  if (skip) return;

  // A mesh kept between runs, when one is being kept and still counts.
  //
  // **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the
  // ones after it. Off by default: a run that is meant to mean something raises from nothing,
  // because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped
  // last week".
  if (warming) {
    const said = await ready(SCENARIO);
    if (said.use === "restore") {
      instanceId = said.instanceId;
      const seconds = await returnTo(instanceId);
      stocked = warmStock(instanceId).images;

      // **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
      // this suite started by hand is gone — the host most of all. Without it the mesh looks
      // perfectly healthy from the control plane's side: a module is assigned, a declaration is
      // sent and recorded, and nothing on the machine is listening to apply it. That is exactly
      // how this was first met, and it cost an hour to see.
      //
      // The real answer is a host started by init, which is what the design says it is anyway
      // (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places
      // it that way, the warm path restarts what it knows it started.
      for (const machine of ["anchor", "laptop"]) {
        await must(machine, `pgrep -x mesh-host >/dev/null || ` +
          `(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
      }
      const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
      assert.equal(running.out.trim(), "yes",
        "the host did not come back after a restore, so nothing would apply anything");

      console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
        `and started the host again`);
      return;
    }
    console.log(`warm: raising fresh — ${said.why}`);
  }

  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
  instanceId = raised.instanceId;

  // The first node raises everything from a file rather than from a bundle built into the binary,
  // because the digests are this registry's and are not known until it is up.
  await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
  await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`);
  stocked = raised.images;
  const first = raised.images[0];
  assert.ok(first, "the scenario stocked no images, so nothing can be mirrored");
  registry = first.slice(0, first.indexOf("/"));

  // A build machine, so anything here can ask the mesh to build something. Placed rather than
  // assumed: nothing else in this scenario would start one.
  if (builder) {
    await incus([
      "file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
      "--mode", "0755",
    ], 180_000);
    await must("anchor", `mkdir -p /var/lib/mesh-builder`);
    await must("anchor",
      `MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
        `MESH_WORKSPACE=/var/lib/mesh-builder ` +
        `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
  }
  if (warming) {
    // Snapshotted only now, with everything up: a state worth returning to is the one after the
    // part nobody wants to repeat.
    await rememberStock(instanceId, stocked);
    const warm = await keep(SCENARIO, instanceId);
    console.log(`warm: ${warm.instanceId} kept, against ` +
      Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
  }
}, { timeout: 1_800_000 });

after(async () => {
  // A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything
  // else is destroyed, because an instance nobody meant to keep is one nobody will remember.
  if (warming) return;
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
  for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) {
    assert.match(running, new RegExp(container), `${container} is not running`);
  }
  // Answering, not merely up. A container that is running is not a control plane that replies —
  // a distinction this project has already paid for once.
  assert.ok((await mesh("status")).length > 0);
});

test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
  for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
    await mesh(`node add ${node}`);
    const token = tokenFrom(await mesh(`token issue --node ${node}`));
    // No --name. The token says what the mesh calls the machine, which is the fault this walk
    // found the first time it was run.
    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
    assert.match(said, new RegExp(`enrolled as ${node}`), said);
    assert.match(said, /sealing key/, "no sealing key was generated");
  }

  const recorded = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select name from node where sealing_key is not null order by name"`);
  assert.equal(recorded.trim().split("\n").map((l) => l.trim()).sort().join(","), "anchor,laptop",
    "the mesh did not record a sealing key for both machines");
});

test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
  // The whole argument, on real machines: the two ends must hold the SAME password, and it must
  // appear nowhere the mesh or the broker could read it.
  await must("anchor", `printf %s '{"module":"postgres","version":"1",` +
    `"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
    `"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
    `"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
  await must("anchor", `printf %s '{"module":"meshboard","version":"1",` +
    `"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
    `"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
    `"secrets":{"postgres-database":"/etc/meshboard/database.password"},"resources":[]}' > /tmp/app.json`);
  await must("anchor", `docker cp /tmp/pg.json mesh-control:/pg.json`);
  await must("anchor", `docker cp /tmp/app.json mesh-control:/app.json`);
  await mesh("module add /pg.json");
  await mesh("module add /app.json");

  await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
  await mesh("overlay place laptop --site lab");
  for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
  await mesh("assign anchor postgres");
  await mesh("assign laptop meshboard");

  for (const machine of ["anchor", "laptop"]) {
    await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
  }
  await mesh("push");
  await new Promise((r) => setTimeout(r, 8000));

  const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
  // Named after the machine *and* the module, because a consumer is both (novox/hq
  // 04-ISSUES/022) — a node routinely runs several modules wanting one database.
  const onProvider = (await must("anchor",
    `cat /var/lib/mesh-host/grants/laptop.meshboard.secret`)).trim();
  assert.ok(onConsumer.length >= 40, `the consumer's credential is ${onConsumer.length} characters`);
  assert.equal(onConsumer, onProvider,
    "the two ends hold different passwords, so nothing could ever authenticate");

  // Only the machine it is for may read it.
  assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.password`), /^600/);

  // And it is nowhere it could have been read on the way. The declaration crossed the broker; the
  // database is the control plane's; the state is what the node reported back.
  for (const [machine, where] of [
    ["laptop", "/var/lib/mesh-host/declared.json"],
    ["laptop", "/var/lib/mesh-host/state.json"],
    ["anchor", "/var/lib/mesh-host/declared.json"],
  ] as const) {
    const { out } = await on(machine, `grep -c ${quote(onConsumer)} ${where}`);
    assert.equal(out.trim(), "0", `the password is in ${where} on ${machine}`);
  }
  const inTheMesh = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select count(*) from secret where for_consumer like '%${onConsumer}%' ` +
      `or for_provider like '%${onConsumer}%'"`);
  assert.equal(inTheMesh.trim(), "0", "the control plane's database holds the password in the clear");
});

test("the consumer is also told where its database is", { skip, timeout: 300_000 }, async () => {
  // A password with no address is not a connection. This is the readable half, which stays
  // readable on purpose — it is the secret half that could not be composed, not this one.
  const told = JSON.parse(await must("laptop", `cat /etc/meshboard/database.json`));
  assert.equal(told.from, "anchor");
  assert.equal(told.at, "anchor.internal");
  assert.equal(told.serves.port, 5432);
  // And the name it resolves to was written by the mesh as well, on this machine.
  assert.match(await must("laptop", `grep anchor.internal /etc/hosts`), /10\.42\.0\.\d+/);
});

test("when a machine cannot do what it was told, the mesh says which and why", { skip, timeout: 900_000 }, async () => {
  // Demonstrated with inserted rows first, which proves the query and not the path. This sends a
  // real machine something it will genuinely fail at, and asks the mesh afterwards.
  //
  // A package that does not exist, because that is a failure of the ordinary kind: the host tries,
  // the package manager says no, and some of the declaration is applied and some is not — which
  // is the situation `status` exists to distinguish from a machine that refused everything.
  await must("anchor", `printf %s '{"module":"impossible","version":"1","resources":[` +
    `{"id":"nothing","type":"package","package":"a-package-that-does-not-exist"}]}' > /tmp/imp.json`);
  await must("anchor", `docker cp /tmp/imp.json mesh-control:/imp.json`);
  await mesh("module add /imp.json");
  await mesh("assign laptop impossible");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 10_000));

  const said = await mesh("status");
  assert.match(said, /not doing what they were told/, said);
  assert.match(said, /laptop/, said);
  // The machine's own words about the resource that failed, not a summary written at this end.
  assert.match(said, /impossible\.nothing/, `the failing resource is not named:\n${said}`);

  // And the distinction survives: this machine FAILED, it did not refuse. Refused means it is
  // exactly as it was; failed means it is in a state nobody declared, and they are fixed in
  // different places.
  assert.match(said, /laptop\s+failed/, said);

  // The other machine is not implicated.
  assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/,
    "a machine that did as it was told is listed as wrong");
});

test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => {
  // A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so
  // a push to one that is not listening must wait rather than vanish. The queue is durable and the
  // message persistent, which ought to be enough — but a lost declaration is silent, and "ought to
  // be" is not a property.
  //
  // The machine is not merely idle here: its host is stopped, so nothing is consuming its queue.

  // Nothing this test asserts should depend on what another left behind. The machine still has a
  // deliberately-impossible module from the test above, and while that is assigned the mesh never
  // updates its account of what the machine holds — a partial report is not an account, on
  // purpose (novox/hq 04-ISSUES/010).
  await mesh("unassign laptop impossible");

  // Stop listening, and prove it stopped — a test that pushed to a machine that was still running
  // would pass having checked nothing.
  //
  // By process name, never by matching the command line: `pkill -f` matches the shell running it
  // too, which kills the connection carrying the command and hangs the caller waiting for a reply
  // that will never come. Cost an hour once, in this file.
  await must("laptop", `pkill -x mesh-host || true; sleep 1`);
  const listening = await on("laptop", `pgrep -x mesh-host`);
  assert.equal(listening.ok, false, "the host is still running, so this proves nothing");

  await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` +
    `{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`);
  await must("anchor", `docker cp /tmp/away.json mesh-control:/away.json`);
  await mesh("module add /away.json");
  await mesh("assign laptop while-away");
  await mesh("push laptop");

  // Nothing has happened on the machine, because nothing is there to do it.
  const before = await on("laptop", `test -f /etc/mesh-while-away`);
  assert.equal(before.ok, false, "a machine with no host applied a declaration");

  // And now it listens again. No second push, and nobody says anything.
  await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`);
  let arrived = false;
  for (let i = 0; i < 20 && !arrived; i++) {
    arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok;
    if (!arrived) await new Promise((r) => setTimeout(r, 2000));
  }
  if (!arrived) {
    // Everything needed to tell "the message was never queued" from "the host never read it".
    const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
    const queues = await on("anchor",
      `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
    const owned = await on("anchor",
      `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
    assert.fail(`a declaration sent to a switched-off machine was lost\n` +
      `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
      `--- what each machine last did ---\n${owned.out}`);
  }
  assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");

  // And the mesh's account of what that machine holds catches up too, or a later declaration
  // would tell it to remove what it has just been given.
  await new Promise((r) => setTimeout(r, 4000));
  const owned = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select owned from node where name = 'laptop'"`);
  assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`);
});

test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => {
  // Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares,
  // and never what the machine raised for itself from its bundle — which is the fault that
  // destroyed a substrate once (novox/hq 04-ISSUES/010).
  //
  // Two modules, so the test can tell "removed the right one" from "removed everything".
  for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) {
    await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` +
      `{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`);
    await must("anchor", `docker cp /tmp/${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
    await mesh(`assign anchor ${name}`);
  }
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 6000));
  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive");
  assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive");

  await mesh("unassign anchor going");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 6000));

  assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false,
    "an unassigned module's file is still there");
  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok,
    "unassigning one module took another one's file with it");

  // And the substrate this machine raised from its own bundle is untouched. It was not declared by
  // the mesh, so the mesh must never remove it — the machine would take its own control plane
  // away, which is exactly what happened before origins existed.
  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
  for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) {
    assert.match(running, new RegExp(container),
      `${container} was removed by a declaration that never declared it`);
  }
});

test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => {
  // The other direction of the same rule. A node that is sent a declaration mentioning none of its
  // private network must not lose it: the network came from a module that is still assigned, and
  // "not in this message" is not "no longer wanted".
  assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok,
    "the private network's configuration is gone");
  assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok,
    "the mesh's names are gone");
});

test("a machine that fell behind catches up without being named", { skip, timeout: 900_000 }, async () => {
  // `status` says which machines are not doing what they were told; something has to act on it.
  // `push --behind` is that something, and it is a command rather than a timer to begin with —
  // a scheduler is then a scheduler over this, rather than a second path to the same act.

  // Break one machine, in a way that is fixable: a package that does not exist yet.
  await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
    `{"id":"pkg","type":"package","package":"a-package-that-does-not-exist-yet"},` +
    `{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
  await must("anchor", `docker cp /tmp/fix.json mesh-control:/fix.json`);
  await mesh("module add /fix.json");
  await mesh("assign laptop fixable");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 8000));

  assert.match(await mesh("status"), /laptop\s+failed/, "the machine did not report a failure");
  // And the resources that COULD be applied were — one broken thing no longer blocks the rest
  // (novox/hq 04-ISSUES/011).
  assert.ok((await on("laptop", `test -f /etc/mesh-fixable`)).ok,
    "a resource after the failing one was never attempted");

  // Nothing is behind on the other machine, so nothing is pushed to it.
  const named = await mesh("push --behind");
  assert.match(named, /laptop/, named);
  assert.doesNotMatch(named, /sent anchor/, `a machine that was fine was pushed to:\n${named}`);

  // Fix the cause, the way somebody would: the module stops asking for the impossible thing.
  await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
    `{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
  await must("anchor", `docker cp /tmp/fix.json mesh-control:/fix.json`);
  await mesh("module add /fix.json");

  // And nobody names the machine.
  await mesh("push --behind");
  await new Promise((r) => setTimeout(r, 8000));

  const after = await mesh("status");
  assert.doesNotMatch(after, /laptop\s+(failed|refused)/,
    `the machine did not recover:\n${after}`);

  // With nothing behind, it says so rather than doing nothing quietly.
  assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
});

test("the mesh runs its own artifact store", {
  skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
  timeout: 900_000,
}, async () => {
  // Artifacts go to a registry, and the only registries that existed were raised by the lab or by
  // the bootstrap bundle. A mesh had no way to run its own.
  //
  // Chicken and egg, resolved the way the bootstrap's is: the scenario's registry serves the image
  // the module mirrors, and the module then runs a registry of the mesh's own.
  await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
    `"provides":[{"name":"artifact-store","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"claims":[{"name":"the-artifact-store","scope":"node"}],` +
    `"serves":{"artifact-store":{"port":5000}},` +
    `"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"${pinned("registry")}"}]},` +
    `"resources":[` +
    `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
    `{"id":"store","type":"container","name":"mesh-registry","artifact":"registry",` +
    `"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
    `> /root/registry/module.json`);
  await must("anchor", `cd /root/registry && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm registry`);

  await mesh("build /root/registry --wait 300s", 420_000);
  await mesh("assign anchor registry");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 12_000));

  // Running, and answering — a container that is up is not a registry that replies.
  assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
  let answers = false;
  for (let i = 0; i < 20 && !answers; i++) {
    answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
    if (!answers) await new Promise((r) => setTimeout(r, 2000));
  }
  assert.ok(answers, "the mesh's own registry is running and does not answer");

  // And reachable from another machine over the private network, which is the whole point of an
  // artifact store being a mesh-scoped provision.
  assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
    "the artifact store is not reachable from another machine, so nothing else can use it");
});

// Defends novox/hq ADR 0007: the mesh is its own certificate authority for internal names.
test("a machine serves its internal name with a certificate the mesh issued", {
  skip, timeout: 900_000,
}, async () => {
  // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
  // Asserted with a real handshake: a certificate that parses and does not chain fails at the
  // moment something connects, which is the worst place to find out.
  await must("anchor", `printf %s '{"module":"served","version":"1",` +
    `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
    `> /tmp/served.json`);
  await must("anchor", `docker cp /tmp/served.json mesh-control:/served.json`);
  await mesh("module add /served.json");
  await mesh("assign anchor served");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 8000));

  assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
  assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");

  // The name it was issued for is the one the mesh gave this machine.
  const named = await must("anchor",
    `openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
    `docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` +
    `"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
  assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);

  // And a real handshake: the machine serves TLS with the key it generated, and another machine
  // verifies it against the mesh's authority and nothing else.
  await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
    `-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
    `> /var/log/tls.log 2>&1 & sleep 2`);
  await must("laptop", `mkdir -p /etc/mesh`);
  const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
  await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);

  const shook = await on("laptop",
    `echo | openssl s_client -connect anchor.internal:8443 ` +
    `-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
  assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` +
    `what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`);
  assert.match(shook.out, /Verification: OK/, shook.out);
});

// Defends novox/hq ADR 0007: what a machine exposes is what its modules declared, and nothing
// arrives at a port nobody asked for.
test("a machine filters exactly what its modules declared, and nothing else", {
  skip, timeout: 900_000,
}, async () => {
  // The rule set is derived from what is assigned, not kept in step by hand — and the proof that
  // matters is not that a file arrived but that packets are treated differently because of it.
  // A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
  //
  // Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
  // so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
  // the shape that rule leaves, and this is the first thing to use it for its real purpose.
  // A listener is written to a file rather than squeezed through three levels of shell quoting.
  // The first attempt did the latter, never started, and the test failed on its own setup —
  // which reads exactly like the firewall working.
  await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` +
    `import socket, sys, threading\n` +
    `def serve(port):\n` +
    `    s = socket.socket()\n` +
    `    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` +
    `    s.bind(("0.0.0.0", port))\n` +
    `    s.listen(8)\n` +
    `    while True:\n` +
    `        c, _ = s.accept()\n` +
    `        c.send(str(port).encode())\n` +
    `        c.close()\n` +
    `for p in (9101, 9102):\n` +
    `    threading.Thread(target=serve, args=(p,), daemon=True).start()\n` +
    `threading.Event().wait()\n` +
    `LISTENER`);
  await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`);

  // Two paths to the same machine, which is what makes "from the mesh" testable at all: over the
  // private network, and over the segment both machines happen to share. A rule that opens a port
  // to the mesh must accept the first and refuse the second — and a test that only ever used one
  // path could not tell "open to the mesh" from "open".
  const reach = async (where: string, port: number) => {
    const said = await on("anchor",
      `timeout 5 python3 -c "import socket;s=socket.create_connection(('${where}',${port}),4);` +
      `print(s.recv(32).decode());s.close()"`);
    return said.ok;
  };
  const overlay = (port: number) => reach("laptop.internal", port);
  const segment = (port: number) => reach("192.0.2.20", port);

  // Reachable both ways before any rule set exists, so what changes afterwards is the rule set and
  // not the listener. Without this the test would pass against a service that never started.
  assert.ok(await overlay(9101), "the declared port never opened, so nothing below tests anything");
  assert.ok(await overlay(9102), "the undeclared port never opened");
  assert.ok(await segment(9101), "the declared port is not reachable off the private network yet, " +
    "so closing it later would prove nothing");

  await must("anchor", `printf %s '{"module":"talker","version":"1",` +
    `"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
    `"resources":[]}' > /tmp/talker.json`);
  // The rule set goes where this machine's nftables unit reads from, and the unit is declared to
  // reflect it. No command anywhere.
  // The module ships the unit that loads its rules, rather than using the one the distribution's
  // nftables package provides. That unit is `Type=oneshot` with no `RemainAfterExit`, so it does
  // its work and goes inactive — and a host asked for a service that is "running" reports, quite
  // correctly, that it is stopped. There is no state in the vocabulary for "ran and exited having
  // done its job", so a module that wants one brings a unit that stays.
  //
  // Which is also the right shape: how a machine enforces rules is a fact about the machine, and
  // the mesh has no business depending on what a distribution happens to package.
  await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
    `"capabilities":["firewall"],` +
    `"filtering":{"into":"/etc/mesh/filter.nft"},` +
    `"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
    `{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"},` +
    `{"id":"unit","type":"file","path":"/etc/systemd/system/mesh-filter.service",` +
    `"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for this machine\\n` +
    `[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
    `ExecStart=/usr/bin/nft -f /etc/mesh/filter.nft\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
    `{"id":"filter","type":"service","unit":"mesh-filter.service","state":"running",` +
    `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
  for (const f of ["talker", "firewall"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign laptop talker");
  await mesh("assign laptop firewall");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 20_000));

  const written = await must("laptop", `cat /etc/mesh/filter.nft`);
  // A rule names its source. Not decoration: it is the only thing that answers "why is this open".
  assert.match(written, /# talker . the thing this test is about/,
    `the rule does not name what caused it:\n${written}`);
  // The mesh's addresses are the ones on the private network, which is what "from the mesh"
  // means — not the segment the machines happen to share.
  assert.match(written, /ip saddr \{ [0-9., ]+ \} tcp dport 9101 accept/,
    `"from the mesh" resolved to nothing:\n${written}`);
  assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);

  // Loaded, not merely written. The service was restarted because a file it reflects changed.
  const table = await must("laptop", `nft list table inet mesh`);
  assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);

  // And it filters. Three assertions, and the third is the one that makes "from the mesh" mean
  // something rather than being a synonym for "open".
  assert.ok(await overlay(9101),
    "the declared port is closed on the private network, so the machine is filtering more than " +
    "it was told to");
  assert.ok(!(await overlay(9102)),
    "a port no module declared is still reachable, so the rule set restricts nothing");
  assert.ok(!(await segment(9101)),
    "the declared port answers off the private network, so `from: mesh` restricted nothing");

  // The machine did not lock itself out of the mesh: it is still taking declarations.
  assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
    "the machine stopped doing what it was told after applying its own rule set");

  // Removing the module that wanted the port closes it, with nobody editing a rule. This is the
  // whole claim of a derived firewall, and it is also the second load — which must replace the
  // table rather than add to it.
  await mesh("unassign laptop talker");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 20_000));
  assert.ok(!(await overlay(9101)),
    "the port stayed open after the module that wanted it was removed");
});

test("the builder is a module the mesh assigns, with a credential the mesh delivered", {
  skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
  timeout: 900_000,
}, async () => {
  // Until this, the builder was a program somebody started on a machine with whatever credential
  // they had to hand — in practice the broker's administrative one. A program documented as
  // holding its own credential and given somebody else's is worse than one with no story at all.
  //
  // So: the mesh issues a scoped account, seals it to the machine, and delivers it with the
  // declaration. Nobody types it and the mesh cannot read it back.
  await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` +
    `"requires":["artifact-store"],"capabilities":["container-runtime"],` +
    `"claims":[{"name":"the-build-machine","scope":"node"}],` +
    `"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` +
    `"own-secrets":{"broker":"/var/lib/mesh/builder/broker"},` +
    `"build":{"artifacts":[{"name":"builder","kind":"upstream",` +
    `"from":"${pinned("mesh-builder")}"}]},` +
    `"resources":[` +
    `{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` +
    `{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` +
    `{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` +
    `"network":"host",` +
    `"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` +
    `"/var/run/docker.sock:/var/run/docker.sock"],` +
    `"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` +
    `"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` +
    `"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`);
  await must("anchor", `cd /root/builder && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm builder`);

  // The builder's own image is built by the builder that is already running — the same
  // chicken-and-egg as the registry, resolved the same way. The one started by hand does this
  // last piece of work and is then replaced by the module it just built.
  await mesh("build /root/builder --wait 300s", 420_000);

  // The mesh makes the account and seals the URL to this machine. Nothing is printed that would
  // work if it were pasted somewhere else.
  const issued = await mesh("builder issue lab-builder --node anchor");
  assert.match(issued, /sealed to anchor/, issued);
  assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/,
    "the credential was printed, so the one copy that matters is on a terminal");

  // Now the hand-started one goes, or two builders race for the same queue and whichever answers
  // proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the
  // connection carrying the command and hangs the caller waiting for a reply that will never
  // come. Cost an hour once, in this file.
  await on("anchor", `pkill -x mesh-builder`);
  await new Promise((r) => setTimeout(r, 2000));
  assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok,
    "the hand-started builder is still running, so this would test that one");

  await mesh("assign anchor builder");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 20_000));

  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
  assert.match(running, /mesh-builder/,
    `the builder was assigned and is not running:\n${running}\n` +
    `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);

  // Running is not connected. A builder that cannot reach the broker sits there, and every
  // outward sign — the container is up, the credential is on disk — says it is working.
  await new Promise((r) => setTimeout(r, 5000));
  const said = await on("anchor", `docker logs mesh-builder 2>&1 | tail -20`);
  assert.doesNotMatch(said.out, /cannot reach the broker/,
    `the builder is running and cannot reach the broker:\n${said.out}`);

  // The credential arrived, is readable only by the machine, and is the scoped account rather
  // than the broker's own.
  assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/);
  const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`);
  assert.match(credential, /"url":"amqps:\/\/lab-builder:/,
    "the builder is using an account that is not its own");
  assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account");
  // And what to check the broker against. A mesh's broker presents a certificate of the mesh's
  // own, so a URL alone reaches only a broker some public authority vouches for — which is no
  // mesh broker at all, and fails at TLS with an error about an unknown authority.
  assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/,
    `the builder was given nothing to verify the broker with:\n${credential}`);

  // And it works: the mesh asks this builder to build something, and it does. Answering is the
  // only proof that the delivered credential authenticates — a container that is up with a
  // credential it cannot use looks identical from outside.
  // Somewhere the builder can actually see. A builder that is a module runs in a container, so
  // the machine's filesystem is not its own — a path like /root only works for a builder somebody
  // started on the host, which is what the first build above used. In a real mesh a module is
  // cloned from the forge over a URL; here it goes in the directory the module already mounts,
  // which is the same fact wearing different clothes.
  const repo = "/var/lib/mesh/builder/repositories/built";
  await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"built","version":"1",` +
    `"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` +
    `> ${repo}/module.json`);
  await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm built`);
  try {
    await mesh(`build ${repo} --wait 300s`, 420_000);
  } catch (why) {
    // The builder's own account of itself. Without it the failure is "nothing consumed the
    // queue", which names no cause and is the same sentence whether the credential was refused,
    // the queue was never declared, or the process died three seconds in.
    const said = (await on("anchor", `docker logs mesh-builder 2>&1 | tail -40`)).out;
    throw new Error(`${(why as Error).message}\n\nwhat the builder said:\n${said}`);
  }

  // Naming the module, and not merely containing its name: `builds` says "nothing has been built
  // yet" when there is nothing, and that sentence contains the word this was matching on.
  const recorded = await mesh("builds built");
  assert.doesNotMatch(recorded, /nothing has been built/,
    `the build was accepted and no build was recorded against the module:\n${recorded}`);
  assert.match(recorded, /built/, recorded);
});

test("rotating a credential moves both ends, and the old one stops working", {
  skip, timeout: 900_000,
}, async () => {
  // The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: on
  // 2026-08-22 a provision documented as never rotating minted a new password on every adoption
  // and updated only the provider's row. Consumers on three nodes held dead credentials for two
  // days while the mesh reported success.
  //
  // So this is checked against a real database with a real login, three times: the delivered
  // credential works, the rotated one works, and the one that was rotated away does not. Two ends
  // holding a matching string proves they agree; only an authentication proves they are right.
  const store = "/var/lib/mesh/postgres";
  await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
    `"provides":[{"name":"real-postgres-database","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"serves":{"real-postgres-database":{"port":5433}},` +
    `"own-secrets":{"superuser":"${store}/superuser"},` +
    `"grants":{"real-postgres-database":"${store}/grants"},` +
    // Both halves. `grants` is where each consumer's sealed password lands; `receives` is the
    // manifest saying who asked and for what. Without the second the provisioner finds a
    // directory of unexplained secrets and says nothing has been granted — which is true, and
    // reads exactly like a credential that was never delivered.
    `"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` +
    `"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
    `"resources":[` +
    `{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
    `{"id":"grants","type":"directory","path":"${store}/grants","mode":"0755"},` +
    `{"id":"postgres-database","type":"container","name":"real-store",` +
    `"image":"${pinned("postgres")}",` +
    `"ports":["5433:5432"],` +
    `"volumes":["${store}/superuser:/run/superuser:ro"],` +
    `"env":{"POSTGRES_PASSWORD_FILE":"/run/superuser"}},` +
    `{"id":"provisioner","type":"container","name":"real-provisioner",` +
    `"image":"${pinned("mesh-provision-postgres")}","network":"host",` +
    `"volumes":["${store}:${store}:ro"],` +
    `"env":{"GRANTS":"${store}/grants",` +
    `"MESH_PROVISION_PASSWORD_FILE":"${store}/superuser",` +
    `"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
    `> /tmp/realstore.json`);
  await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
    `"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` +
    `"binds":{"real-postgres-database":"/etc/realapp/where.json"},` +
    `"secrets":{"real-postgres-database":"/etc/realapp/password"},` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
    `> /tmp/realapp.json`);
  for (const f of ["realstore", "realapp"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign anchor realstore");
  await mesh("assign laptop realapp");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 30_000));

  // A real login from the consumer's machine, over the private network — not over loopback, where
  // pg_hba trusts anything and every password looks correct. That was done here once and the test
  // passed for an afternoon while verifying nothing: a deliberately wrong password returned a row.
  // As the role the provisioner made, into the database it made. The provisioner names a role
  // after the machine and a database after what the module asked for — which is the contract, and
  // getting it wrong here made the test fail against a provisioner that had done its job.
  // By address, resolved on the machine.
  //
  // **This container is not the mesh's.** The mesh gives its names to the containers it declares,
  // and this one is started by the test with `docker run` — nothing declared it, so nothing
  // configured it. That boundary is the right one: a container somebody runs by hand is not the
  // mesh's to configure, and reaching into every container on a machine is what a resolver in
  // resolv.conf would be for.
  //
  // So the workaround stays here, and the proof that names work inside containers is its own
  // test, against a container the mesh declared.
  const where = (await must("laptop",
    `getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
  assert.match(where, /^[0-9.]+$/, `the mesh's name for anchor does not resolve here: ${where}`);

  const login = async (password: string) =>
    await on("laptop", `docker run --rm -e PGPASSWORD=${quote(password)} ` +
      // The role the provisioner made: mesh_<node>_<module>, because a consumer is a module on
      // a machine (novox/hq 04-ISSUES/022).
      `${pinned("postgres")} psql -h ${where} -p 5433 -U mesh_laptop_realapp ` +
      `-d realapp -qAt -c "select 1"`, 120_000);

  const diagnostics = async () =>
    `provisioner:\n${(await on("anchor", `docker logs real-provisioner 2>&1 | tail -20`)).out}\n` +
    `grants:\n${(await on("anchor", `ls -l ${store}/grants`)).out}`;

  const first = (await must("laptop", `cat /etc/realapp/password`)).trim();
  assert.ok(first.length >= 40, `the consumer's credential is ${first.length} characters`);
  let works = false;
  for (let i = 0; i < 20 && !works; i++) {
    works = (await login(first)).ok;
    if (!works) await new Promise((r) => setTimeout(r, 5000));
  }
  assert.ok(works, `the delivered credential does not authenticate:\n` +
    `${(await login(first)).out}\n${await diagnostics()}`);

  // Now rotate. One command: the record changes AND both ends are sent, because leaving the
  // sending to a later command is the fault above, exactly.
  const said = await mesh("rotate real-postgres-database", 180_000);
  assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
  assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
  await new Promise((r) => setTimeout(r, 25_000));

  const second = (await must("laptop", `cat /etc/realapp/password`)).trim();
  assert.notEqual(second, first, "the consumer was handed back the credential just rotated away");

  // The new one authenticates — the only proof the provider was told the same thing the consumer
  // was given. Two files agreeing proves they agree, not that either is right.
  let now = false;
  for (let i = 0; i < 20 && !now; i++) {
    now = (await login(second)).ok;
    if (!now) await new Promise((r) => setTimeout(r, 5000));
  }
  assert.ok(now, `after rotation the new credential does not authenticate, so the two ends ` +
    `disagree — which is the fault this exists to make impossible:\n${await diagnostics()}`);

  // And the old one does not. Without this the test passes against a provider that added a
  // password without replacing one, which is a rotation that rotates nothing.
  assert.ok(!(await login(first)).ok,
    "the password that was rotated away still authenticates, so nothing was rotated");
});

test("a route is a grant: a workload is reached by the name it asked for", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq 08-connectivity §3. The mirror of a database grant: there the consumer supplies a
  // name and receives credentials; here it supplies a target and receives a name. Nothing new in
  // the vocabulary — a route is a provision like any other.
  //
  // The workload is the registry image, because it is an HTTP server this scenario already has.
  // What is being tested is the mesh's arrangement, not the workload.
  await must("anchor", `printf %s '{"module":"frontdoor","version":"1",` +
    `"provides":[{"name":"route","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"receives":{"route":"/etc/frontdoor/routes.json"},` +
    `"serves":{"route":{"domain":"mesh.test"}},` +
    `"listens":[{"port":8081,"from":"mesh","why":"the front door"}],` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/frontdoor","mode":"0755"},` +
    `{"id":"proxy","type":"container","name":"front-door",` +
    `"image":"${pinned("mesh-route-proxy")}","network":"host",` +
    `"volumes":["/etc/frontdoor:/etc/frontdoor:ro"],` +
    `"env":{"ROUTES":"/etc/frontdoor/routes.json","LISTEN":":8081"}}]}' ` +
    `> /tmp/frontdoor.json`);
  // The workload declares the port it listens on as well as the route it wants. Both, because
  // they are different questions: one says who may reach it, the other says by what name — and
  // the earlier test left this machine filtering, so a module that asked for a route and not for
  // the port would be unreachable by the proxy it just asked for.
  await must("anchor", `printf %s '{"module":"storefront","version":"1",` +
    `"requires":["route"],"capabilities":["container-runtime"],` +
    `"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
    `"binds":{"route":"/etc/storefront/route.json"},` +
    `"listens":[{"port":8088,"from":"mesh","why":"the proxy reaches it here"}],` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/storefront","mode":"0755"},` +
    `{"id":"app","type":"container","name":"storefront",` +
    `"image":"${pinned("registry")}","ports":["8088:5000"]}]}' > /tmp/storefront.json`);
  for (const f of ["frontdoor", "storefront"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign anchor frontdoor");
  await mesh("assign laptop storefront");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 25_000));

  // The provider was told who asked, and where that machine is — which it needs in order to
  // reach back, and which it must not have to derive from a naming convention.
  const routes = await must("anchor", `cat /etc/frontdoor/routes.json`);
  assert.match(routes, /shop\.mesh\.test/, `the proxy was not told about the route:\n${routes}`);
  assert.match(routes, /"at": *"laptop\.internal"/,
    `the proxy was not told where the consumer is, so it cannot reach it:\n${routes}`);

  // And the consumer was told what the provider serves, which is how it knows its own name.
  const bound = await must("laptop", `cat /etc/storefront/route.json`);
  assert.match(bound, /mesh\.test/, `the consumer was not told the public name:\n${bound}`);

  // The whole point: a request for the name reaches the workload, across the private network.
  let reached = false;
  let said = "";
  for (let i = 0; i < 20 && !reached; i++) {
    const answer = await on("anchor",
      `curl -sf -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
    said = answer.out;
    reached = answer.ok && said.trim() === "200";
    if (!reached) await new Promise((r) => setTimeout(r, 4000));
  }
  assert.ok(reached, `a request for the name did not reach the workload (${said}):\n` +
    `${(await on("anchor", `docker logs front-door 2>&1 | tail -20`)).out}`);

  // Withdrawal, which 08-connectivity lists as open: a stale public name pointing at nothing
  // fails more visibly than a stale grant, so it must not survive the module leaving.
  await mesh("unassign laptop storefront");
  await mesh("push");

  // **Waited for, not slept through.** The mesh withdrawing a route and the machine acting on it
  // are different things, and a fixed sleep between them tests whichever the clock happened to
  // land on — this assertion passed twice and failed once on nothing but timing.
  //
  // A machine that has not applied yet is also not a machine that failed, so `status` cannot
  // stand in for this: "not yet" and "never" look identical there, and only one of them is worth
  // failing over.
  let after = "";
  let withdrawn = false;
  for (let i = 0; i < 20 && !withdrawn; i++) {
    after = await must("anchor", `cat /etc/frontdoor/routes.json`);
    withdrawn = !after.includes("shop.mesh.test");
    if (!withdrawn) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(withdrawn,
    `the route outlived the module that asked for it:\n${after}\n\n` +
    `the machine did apply — this is what the mesh would send now:\n` +
    `${(await on("anchor", `docker exec mesh-control /mesh-control plan anchor --files`)).out}`);

  let gone = false;
  for (let i = 0; i < 15 && !gone; i++) {
    const answer = await on("anchor",
      `curl -s -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
    gone = answer.out.trim() === "404";
    if (!gone) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(gone, "the proxy still serves a name whose module was unassigned");
});

test("model access is answered by a record, and the key the mesh took is one it cannot read", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq ADR 0024. The first provision no machine answers: a hosted model is on nobody's
  // node and is reached over the public internet, so the rule that refuses two ends sharing no
  // private network must not apply to it.
  await must("anchor", `printf %s '{"module":"assistant","version":"1",` +
    `"requires":["model-access"],` +
    `"binds":{"model-access":"/etc/assistant/model.json"},` +
    `"secrets":{"model-access":"/etc/assistant/key"},` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/assistant","mode":"0755"}]}' ` +
    `> /tmp/assistant.json`);
  await must("anchor", `docker cp /tmp/assistant.json mesh-control:/assistant.json`);
  await mesh("module add /assistant.json");

  // The licences first. With none recorded at all the honest answer is that nothing provides
  // model-access — correct, and a different refusal from the one being tested.
  await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`);
  await mesh(`licence add anthropic the-organisation --serves '{"model":"a-model"}'`);

  // Refused at the earliest point somebody could meet it: assigning records the assignment and
  // then says the machine's set cannot be applied. The refusal names both candidates and the
  // command. ADR 0024 warns this will be felt — which is correct, and correct is not the same as
  // usable.
  const refused = await on("anchor", `docker exec mesh-control /mesh-control assign laptop assistant`);
  assert.ok(!refused.ok,
    `a consumer was given model access without anybody saying which:\n${refused.out}`);
  for (const want of ["personal", "the-organisation", "licence use"]) {
    assert.match(refused.out, new RegExp(want),
      `the refusal does not name ${want}:\n${refused.out}`);
  }

  await mesh("licence use personal laptop assistant");

  // Chosen, and still no key: the mesh has one thing to deliver and has not been given it.
  const noKey = await on("anchor", `docker exec mesh-control /mesh-control plan laptop`);
  assert.ok(!noKey.ok, `a module was planned with a licence that has no key:\n${noKey.out}`);
  assert.match(noKey.out, /licence key personal/, noKey.out);

  // The accept verb. Given on standard input rather than as an argument, because a key in a
  // command line is a key in shell history and in every process listing taken while it ran.
  const secret = "sk-test-" + "0123456789abcdef".repeat(2);
  const accepted = await must("anchor",
    `printf %s ${quote(secret)} | docker exec -i mesh-control /mesh-control licence key personal`);
  assert.match(accepted, /sealed to 1 holder/, accepted);
  assert.doesNotMatch(accepted, new RegExp(secret),
    "the key was echoed back, so the one copy that matters is on a terminal");

  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));

  // What is public arrives, and says it is a record rather than leaving an empty address that a
  // reader would take for something the mesh failed to fill in.
  const bound = await must("laptop", `cat /etc/assistant/model.json`);
  assert.match(bound, /personal/, `the consumer was not told which licence it is on:\n${bound}`);
  assert.match(bound, /a-model/, `what the licence serves did not arrive:\n${bound}`);
  assert.match(bound, /not a machine/, `the binding leaves an unexplained empty address:\n${bound}`);

  // And the key arrives, readable only by this machine.
  assert.equal((await must("laptop", `cat /etc/assistant/key`)).trim(), secret,
    "the key that arrived is not the key that was given");
  assert.match(await must("laptop", `stat -c %a /etc/assistant/key`), /^600/);

  // The mesh cannot read it back. This is the whole argument: what is stored is unusable by
  // whoever holds it, the control plane included.
  const stored = await must("anchor",
    `docker exec mesh-store psql -U postgres -d licences -qAt ` +
    `-c "select coalesce(sealed,'') from licence_holder"`);
  assert.ok(stored.trim().length > 0, "nothing was stored, so nothing was sealed");
  assert.doesNotMatch(stored, new RegExp(secret),
    "the key is in the control plane's own database in the open");

  // Nor is it anywhere it could have been read on the way.
  for (const where of ["/var/lib/mesh-host/declared.json", "/var/lib/mesh-host/state.json"]) {
    // Whether grep found it, not how many times. `grep -c` prints 0 and exits non-zero when it
    // finds nothing, so the obvious `|| echo 0` prints a second one and the count is never what
    // it looks like.
    const found = await on("laptop", `grep -q ${quote(secret)} ${where}`);
    assert.ok(!found.ok, `the key is in the open in ${where}`);
  }
});

test("a new commit reaches a machine that is already running the old one", {
  skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
  timeout: 900_000,
}, async () => {
  // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
  // question "did my change go out?". This is that question, end to end — a commit, a build, a
  // catalogue, and a machine that ends up running what the source says.
  const repo = "/var/lib/mesh/builder/repositories/delivered";
  const write = async (what: string) =>
    await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
      `"resources":[{"id":"marker","type":"file","path":"/etc/delivered",` +
      `"content":"${what}","mode":"0644"}]}' > ${repo}/module.json`);

  await write("first");
  await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm first`);
  await mesh(`build ${repo} --wait 300s`, 420_000);
  await mesh("assign laptop delivered");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first");

  // Nothing has moved, so nothing is behind — and it says so rather than doing nothing quietly.
  assert.match(await mesh("build --behind"), /every module the mesh holds is what its source last had/);

  // Now the source moves.
  await write("second");
  await must("anchor", `cd ${repo} && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm second`);
  const moved = (await must("anchor", `cd ${repo} && git rev-parse HEAD`)).trim();
  await mesh(`module moved delivered ${moved}`);

  // The mesh says which module is behind, and by how much, before anything is built.
  const behind = await mesh("status");
  assert.match(behind, /delivered/, `status does not name the module that moved:\n${behind}`);
  assert.match(behind, /build --behind/, `status does not say how to catch up:\n${behind}`);

  // The loop, in one command: everything behind its source is built and recorded.
  const built = await mesh("build --behind --wait 300s", 420_000);
  assert.match(built, /delivered/, built);

  // The machine is still running the old one until it is told — the mesh changing its mind is
  // not the same as a machine acting on it, and collapsing the two is how a mesh reports success
  // for something that has not happened.
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first",
    "the machine changed before anything was sent to it");

  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "second",
    "the machine is still running what the source no longer says");

  // And it is no longer out of date, which is the half that makes the answer trustworthy: a
  // status that says "behind" for ever is one nobody reads.
  const after = await mesh("status");
  assert.doesNotMatch(after, /delivered.*<.*[0-9a-f]{8}/,
    `the module is still reported as behind after catching up:\n${after}`);
});

test("the board names the machine that is not doing what it was told", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq 03-DESIGN/01-to-be/11-a-board.md. The board being replaced reads every context's
  // database directly; this one asks the same questions through the same functions the commands
  // use, and holds nothing. So the check is that what it says matches what the mesh says, and
  // that it says the thing a person opened it for.
  await must("anchor", `printf %s '{"module":"board","version":"1",` +
    `"listens":[{"port":8090,"from":"mesh","why":"the board"}],` +
    `"resources":[]}' > /tmp/board.json`);
  await must("anchor", `docker cp /tmp/board.json mesh-control:/board.json`);
  await mesh("module add /board.json");

  // Served from the control plane's own container, reading the mesh on every request.
  await must("anchor", `docker exec -d mesh-control /mesh-control board --listen 0.0.0.0:8090`);
  await new Promise((r) => setTimeout(r, 3000));

  const read = async (path: string) =>
    await on("anchor", `curl -sf http://127.0.0.1:8090${path}`, 30_000);

  let up = false;
  let said = { out: "", ok: false };
  for (let i = 0; i < 15 && !up; i++) {
    said = await read("/");
    up = said.ok;
    if (!up) await new Promise((r) => setTimeout(r, 2000));
  }
  assert.ok(up, `the board does not answer:\n${said.out}`);

  // Something a person opened it for: give a machine a declaration it cannot apply.
  //
  // A unit that does not exist, because the host says so immediately and in its own words. A file
  // in a missing directory is not impossible — the host creates the parents, which is correct and
  // made the first version of this test break nothing at all.
  await must("anchor", `printf %s '{"module":"impossible","version":"1",` +
    `"resources":[{"id":"nowhere","type":"service","unit":"nothing-like-this.service",` +
    `"state":"running"}]}' > /tmp/impossible.json`);
  await must("anchor", `docker cp /tmp/impossible.json mesh-control:/impossible.json`);
  await mesh("module add /impossible.json");
  await mesh("assign laptop impossible");
  await mesh("push laptop");

  let named = false;
  let page = "";
  for (let i = 0; i < 20 && !named; i++) {
    page = (await read("/")).out;
    named = page.includes("laptop") && page.includes(">failed<");
    if (!named) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(named, `the board does not name the machine that failed:\n${page}`);

  // The host's own words, which say exactly what it could not do. A board that said only
  // "failed" would make a person go and ask the thing they opened the board to avoid asking.
  assert.match(page, /nowhere/, `the board does not say what failed:\n${page}`);

  // It agrees with the command, because both read the same thing. Two answers to "which machine
  // is broken" is worse than either alone.
  const asJSON = JSON.parse((await read("/mesh.json")).out);
  assert.equal(asJSON.wrong[0].node, "laptop", `the page and the JSON disagree: ${JSON.stringify(asJSON)}`);
  assert.equal(asJSON.wrong[0].outcome, "failed");

  const fromCommand = JSON.parse(await mesh("status --json"));
  assert.deepEqual(asJSON.wrong, fromCommand.wrong,
    "the board and the command disagree about which machine is broken");

  // And it changes nothing: the mesh is exactly as it was after being read.
  const before = await mesh("status");
  await read("/");
  assert.equal(await mesh("status"), before, "reading the board changed the mesh");

  await mesh("unassign laptop impossible");
  await mesh("push laptop");
});

// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
test("the hub can be filtered without severing the mesh", {
  skip, timeout: 900_000,
}, async () => {
  // The machine that most needs a firewall was the one that could not have one. A hub is dialled
  // by every node at other sites; a machine that is not a hub dials out and needs nothing open.
  // They are the same module, so a static `listens` cannot say it — and the machine it gets wrong
  // is the one facing the public internet.
  //
  // The failure this guards against is not subtle and is very hard to recover from: a rule set
  // that closes the hub's own port takes the private network down, and the mesh's way of fixing
  // anything is to send a declaration over it.
  // Its own directory. Another module on this machine already declares /etc/mesh, and the mesh
  // refuses two modules declaring one path rather than letting the second quietly win — which it
  // did here, correctly, the first time this ran.
  const rules = "/etc/mesh-hub/filter.nft";
  await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` +
    `"capabilities":["firewall"],` +
    `"filtering":{"into":"${rules}"},` +
    `"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
    `{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` +
    `{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` +
    `"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` +
    `[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
    `ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
    `{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` +
    `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`);
  await must("anchor", `docker cp /tmp/hubfilter.json mesh-control:/hubfilter.json`);
  await mesh("module add /hubfilter.json");
  await mesh("assign anchor hubfilter");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 20_000));

  // The hub's own way onto the private network is open, and derived — nothing in that manifest
  // mentions a port.
  const written = await must("anchor", `cat ${rules}`);
  assert.match(written, /udp dport 51820 accept/,
    `the hub's rule set closes the private network it is the way onto:\n${written}`);
  // The module that provides the private network, not the requirement it answers: `networking`
  // is the domain a module offers, and what caused a rule is the module itself.
  assert.match(written, /# mesh-wireguard — the private network/,
    `the rule does not name what caused it:\n${written}`);

  // Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if
  // the overlay is severed. This is the assertion that matters — a rule file that looks right and
  // a mesh that has stopped are exactly what this is guarding against.
  assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/);

  await must("laptop", `rm -f /etc/mesh-still-works`);
  await must("anchor", `printf %s '{"module":"stillworks","version":"1",` +
    `"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` +
    `"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`);
  await must("anchor", `docker cp /tmp/stillworks.json mesh-control:/stillworks.json`);
  await mesh("module add /stillworks.json");
  await mesh("assign laptop stillworks");
  await mesh("push laptop");

  let arrived = false;
  for (let i = 0; i < 20 && !arrived; i++) {
    arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok;
    if (!arrived) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(arrived,
    "the hub applied its own rule set and the mesh stopped reaching the other machine");

  // And the other machine still reaches the hub over the private network, which is what the
  // opened port is for.
  assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok,
    "the private network is down after the hub filtered itself");

  await mesh("unassign anchor hubfilter");
  await mesh("unassign laptop stillworks");
  await mesh("push");
});

test("a container reaches another machine by the name the mesh gave it", {
  skip, timeout: 900_000,
}, async () => {
  // Internal names are written to the machine's hosts file, which serves the machine and not what
  // the machine runs: a container gets its own hosts file holding only its own hostname. So every
  // name the mesh wrote was invisible to the majority of things that need one.
  //
  // Checked from inside a container rather than on the machine, because on the machine it has
  // always worked — and that is exactly what made this easy to miss.
  await must("anchor", `printf %s '{"module":"resolves","version":"1",` +
    `"capabilities":["container-runtime"],` +
    `"resources":[{"id":"idle","type":"container","name":"resolves",` +
    `"image":"${pinned("registry")}"}]}' > /tmp/resolves.json`);
  await must("anchor", `docker cp /tmp/resolves.json mesh-control:/resolves.json`);
  await mesh("module add /resolves.json");
  await mesh("assign laptop resolves");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));

  // The names are in the container's own hosts file, written by the runtime.
  const inside = await must("laptop", `docker exec resolves cat /etc/hosts`);
  assert.match(inside, /anchor\.internal/,
    `the container cannot see the other machine's name:\n${inside}`);
  assert.match(inside, /laptop\.internal/,
    `the container cannot see its own machine's name:\n${inside}`);

  // And the name actually reaches the machine, which is the part that matters: a hosts entry
  // pointing at the wrong address resolves perfectly and connects to nothing.
  const reached = await on("laptop",
    `docker exec resolves sh -c 'getent hosts anchor.internal'`);
  assert.ok(reached.ok, `the name does not resolve inside the container:\n${reached.out}`);
  const address = reached.out.trim().split(/\s+/)[0];
  const onTheMachine = (await must("laptop",
    `getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
  assert.equal(address, onTheMachine,
    "the container and its machine disagree about where the other machine is");

  await mesh("unassign laptop resolves");
  await mesh("push laptop");
});

// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
// told each one.
test("every name under a machine resolves to that machine", {
  skip, timeout: 900_000,
}, async () => {
  // Services are named under the machine they run on — postgres.novox.internal,
  // plex.ace.internal. The first label is the service and the rest is the node, so what must
  // resolve is anything under a node's name. What routes it once it arrives is a proxy's, and
  // stays separate.
  //
  // The mesh writes the data and runs no daemon: a resolver is third-party software, and the
  // mesh has no business choosing one. So what is checked here is the mesh's half — that the
  // data is right, complete, and follows the machines.
  await mesh("assign anchor mesh-resolver");
  await mesh("assign laptop mesh-resolver");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 15_000));

  for (const machine of ["anchor", "laptop"]) {
    const written = await must(machine, `cat /etc/mesh-resolver/nodes.conf`);

    // A wildcard per machine, matching the name and everything under it. Both machines get the
    // whole mesh: a node resolves every other node, and itself.
    for (const node of ["anchor", "laptop"]) {
      assert.match(written, new RegExp(`address=/${node}\\.internal/10\\.42\\.0\\.\\d+`),
        `${machine} cannot resolve names under ${node}:\n${written}`);
    }

    // And the addresses agree with what the machine's own hosts file says. Two accounts of where
    // a machine is, disagreeing, would be worse than either alone — and this is the one place
    // they could drift, because they are generated separately.
    const hosts = await must(machine, `getent hosts anchor.internal | head -1 | cut -d' ' -f1`);
    assert.match(written, new RegExp(`address=/anchor\\.internal/${hosts.trim().replace(/\./g, "\\.")}`),
      `the resolver data and the hosts file disagree about where anchor is:\n${written}`);
  }

  // It follows the machines. A node leaving the private network must stop being answered for,
  // because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
  // fails at once and says which name it was.
  //
  // Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the
  // network, so unassigning the domain module alone leaves the machine on the network — pulled
  // back by its own requirement. The mesh was right and this test was wrong the first time.
  await mesh("unassign laptop mesh-resolver");
  await mesh("unassign laptop networking");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 15_000));

  const after = await must("anchor", `cat /etc/mesh-resolver/nodes.conf`);
  assert.doesNotMatch(after, /address=\/laptop\.internal\//,
    `a machine that left the private network is still answered for:\n${after}`);
  assert.match(after, /address=\/anchor\.internal\//,
    `the machine that stayed lost its own name:\n${after}`);

  await mesh("assign laptop networking");
  await mesh("unassign anchor mesh-resolver");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 15_000));
});

test("a service is reached by a name under the machine it runs on", {
  skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-control's examples/modules" : false),
  timeout: 900_000,
}, async () => {
  // postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
  // the node, so anything under a node's name must resolve to that node. What routes it once it
  // arrives is a proxy's concern and stays separate.
  //
  // The mesh writes the data; a module runs the daemon. Both manifests are read from the
  // repository rather than written here, so what is proven is what ships.
  // `resolved-split-dns`, not `resolv-conf`: these machines run systemd-resolved, which owns
  // /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
  // refusal rather than a fight over the file — and picking the wrong one here would have been
  // testing that fight.
  for (const name of ["dnsmasq", "resolved-split-dns"]) {
    const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8");
    await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`);
    await must("anchor", `docker cp /tmp/${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
  }

  // Both machines, because a node resolves from its own copy — the same rule as everything else
  // it holds. A mesh where one machine answers for all of them stops resolving when that machine
  // does, which is the arrangement this design refuses everywhere else.
  for (const machine of ["anchor", "laptop"]) {
    await mesh(`assign ${machine} dnsmasq`);
    await mesh(`assign ${machine} resolved-split-dns`);
  }
  await mesh("push");
  await new Promise((r) => setTimeout(r, 25_000));

  // Everything this test could want to know, gathered in one place.
  //
  // Three times now a diagnostic has not run because the thing before it threw: `must` on a
  // command that fails, and then a query that hangs long enough to take the harness's own timeout
  // with it. A 30-second test costs fifteen minutes to re-run, so the evidence has to be gathered
  // whether the failure is an assertion, an error, or a hang.
  const diagnose = async (machine: string) =>
    `--- ${machine}\n` +
    `dnsmasq: ${(await on(machine, `systemctl is-active dnsmasq.service`)).out.trim()}\n` +
    `${(await on(machine, `journalctl -u dnsmasq -n 12 --no-pager`)).out}\n` +
    `resolved: ${(await on(machine, `resolvectl status | head -30`)).out}\n` +
    `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
    `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
    `listening:\n${(await on(machine, `ss -lnup | grep :53 || echo none`)).out}\n` +
    `asked directly:\n${(await on(machine,
      `timeout 5 resolvectl query postgres.anchor.internal 2>&1 || echo "no answer"`)).out}`;

  // `on`, not `must`: `is-active` exits non-zero for a unit that failed, so `must` would throw
  // before the assertion below — taking every diagnostic with it. That happened, and the run said
  // only "failed".
  for (const machine of ["anchor", "laptop"]) {
    const state = await on(machine, `systemctl is-active dnsmasq.service`);
    if (state.out.trim() === "active") continue;
    assert.fail(`the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
      `its config:\n${(await on(machine, `cat /etc/dnsmasq.conf`)).out}\n` +
      `${await diagnose(machine)}`);
  }

  // Through the machine's own resolver, by the path an application actually takes: nsswitch, then
  // files, then DNS. `dig` would ask a server directly and prove less — the resolv.conf module is
  // half of what is being tested, and only this path goes through it.
  //
  // Bounded on the machine rather than by the harness: a query that hangs is a result, and letting
  // it run into the harness's own timeout turns it into an error with no evidence attached.
  const resolves = async (machine: string, name: string) => {
    const said = await on(machine,
      `timeout 5 getent hosts ${name} | head -1 | cut -d' ' -f1`, 20_000);
    return said.out.trim();
  };
  const addressOf = async (machine: string, node: string) =>
    (await must(machine, `getent hosts ${node}.internal | head -1 | cut -d' ' -f1`)).trim();

  const anchorAt = await addressOf("anchor", "anchor");
  const laptopAt = await addressOf("anchor", "laptop");

  // A name the mesh was never told about, under a machine it was — from both machines, because a
  // node must answer for every machine and not only for itself.
  for (const machine of ["anchor", "laptop"]) {
    let got = "";
    for (let i = 0; i < 15 && !got; i++) {
      got = await resolves(machine, "postgres.anchor.internal");
      if (!got) await new Promise((r) => setTimeout(r, 3000));
    }
    assert.equal(got, anchorAt,
      `${machine} does not resolve a service named under anchor: ${got || "(nothing)"}\n\n` +
      `${await diagnose(machine)}`);
  }

  // Any name at all, which is the whole point: the mesh was never told these exist.
  for (const [name, expected] of [
    ["postgres-2.anchor.internal", anchorAt],
    ["keycloak.anchor.internal", anchorAt],
    ["plex.laptop.internal", laptopAt],
    ["radarr.laptop.internal", laptopAt],
  ] as const) {
    assert.equal(await resolves("laptop", name), expected,
      `${name} did not resolve to the machine it is named under\n\n${await diagnose("laptop")}`);
  }

  // The machine's own name still resolves, and to the same place. Two accounts of where a machine
  // is, disagreeing, would be worse than either alone.
  assert.equal(await resolves("laptop", "anchor.internal"), anchorAt);

  // And what is not the mesh's is not answered by it. The resolver takes over the mesh's names
  // and nothing else, which is what lets a machine keep whatever DNS it already had.
  assert.equal(await resolves("anchor", "something.example.com"), "",
    "the resolver answered for a name that is not the mesh's");

  for (const machine of ["anchor", "laptop"]) {
    await mesh(`unassign ${machine} resolved-split-dns`);
    await mesh(`unassign ${machine} dnsmasq`);
  }
  await mesh("push");
});

// A real third-party workload, adopted the way the conversion will adopt one.
//
// **Everything before this used modules written to exercise the mesh.** This one is software
// nobody here wrote, taking its credentials the way such software does — from its environment —
// and needing two containers that reach each other by name. It is the first module that could not
// have been declared before today: it needs the `network` shape, and it needs a sealed value to
// reach a container's environment.
//
// Its database password is **accepted rather than generated**, which is the whole shape of an
// adoption: a service that already exists keeps the credential it already has, because minting a
// new one is how a running application stops being able to reach its own database.
test("a third-party workload is adopted, with the credential it already had", {
  skip, timeout: 900_000,
}, async () => {
  const password = "the-password-it-already-had";

  await must("anchor", `printf %s ${quote(JSON.stringify({
    module: "umami",
    version: "1",
    capabilities: ["container-runtime"],
    "own-secrets": {
      database: "/var/lib/umami/database.env",
      app: "/var/lib/umami/app.env",
    },
    listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }],
    resources: [
      { id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" },
      // The two containers must reach each other by name, which is what this shape is for.
      { id: "net", type: "network", name: "umami" },
      {
        id: "db", type: "container", name: "umami-db",
        image: pinned("postgres"),
        network: "umami",
        env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" },
        "env-file": ["/var/lib/umami/database.env"],
      },
      {
        id: "app", type: "container", name: "umami",
        image: pinned("ghcr.io/umami-software/umami"),
        network: "umami",
        env: { DATABASE_TYPE: "postgresql" },
        "env-file": ["/var/lib/umami/app.env"],
        ports: ["1212:3000"],
      },
    ],
  }))} > /umami.json`);
  await must("anchor", `docker cp /umami.json mesh-control:/umami.json`);
  await mesh("module add /umami.json");

  // **Accepted, not generated.** The value is what the database already answers to; the mesh
  // seals it and cannot read it again. Given whole, as the environment lines the containers read.
  await must("anchor",
    `printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
    `docker exec -i mesh-control /mesh-control secret accept anchor umami database --from -`);
  await must("anchor",
    `printf %s ${quote(
      `DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
    `docker exec -i mesh-control /mesh-control secret accept anchor umami app --from -`);

  await mesh("assign anchor umami");
  await mesh("push anchor", 300_000);

  // Both containers, and the network they share.
  let up = false;
  for (let i = 0; i < 60 && !up; i++) {
    const running = await on("anchor", `docker ps --format '{{.Names}}'`);
    up = running.out.includes("umami-db") && running.out.includes("umami");
    if (!up) await new Promise((r) => setTimeout(r, 5000));
  }
  if (!up) {
    // Everything that could say why, gathered before asserting. "It did not start" is the one
    // thing already known; what is wanted is whether the mesh sent it, whether the host refused
    // it, and what the runtime said when it tried.
    const said = await mesh("status");
    const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`);
    const applied = await on("anchor",
      `${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`);
    const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` +
      `for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`);
    const tried = await on("anchor",
      `docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` +
      `docker logs umami-db 2>&1 | tail -15`);
    assert.fail(
      `the workload never started.\n\n` +
      `── what the mesh thinks:\n${said}\n` +
      `── containers:\n${containers.out}\n` +
      `── what the host owns:\n${applied.out}\n` +
      `── what the mesh wrote:\n${files.out}\n` +
      `── the database container:\n${tried.out}\n`);
  }

  // The environment file the mesh sealed is on the machine and readable only by root.
  const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`);
  assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root");

  // **The assertion that matters: the credential works.** Not that a file arrived — that the
  // database the mesh started answers to the password the mesh was given rather than one it made.
  let connected = { out: "", ok: false };
  for (let i = 0; i < 40 && !connected.ok; i++) {
    connected = await on("anchor",
      `docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`);
    if (!connected.ok) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(connected.ok, `the database never came up:\n${connected.out}`);

  const wrong = await on("anchor",
    `docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` +
    `psql -h umami-db -U umami -d umami -qAt -c 'select 1'`);
  assert.ok(!wrong.ok,
    "the database accepted a password nobody gave it, so this proves nothing about the one that was");

  // And the two containers reach each other by name over the module's own network.
  const reached = await must("anchor",
    `docker run --rm --network umami ${pinned("postgres")} ` +
    `sh -c 'getent hosts umami-db || echo unreachable'`);
  assert.doesNotMatch(reached, /unreachable/,
    "a container could not reach the other by name, so the module's network did nothing");
});

// The real modules, resolved together on one machine.
//
// **What this proves without pulling a gigabyte of images**: that five manifests written from the
// running system resolve as a graph — keycloak's requirement met by postgres's provision,
// capabilities checked, nothing claiming the same singular thing — and that the declaration the
// control plane composes is one the host accepts. `plan --json` exists for exactly this: it is
// the only way to know that what the control plane emits is what the host takes.
//
// Running them needs their images stocked and two provisioners built, which is a separate and
// larger job. This is the half that can be known now, and it is the half where a design fault
// would live.
test("the real modules resolve together, and compose a declaration a host accepts", {
  skip, timeout: 300_000,
}, async () => {
  const modules = ["postgres", "keycloak", "gitea", "minio", "mailu"];
  for (const name of modules) {
    const raw = readFileSync(
      `${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
    await must("anchor", `printf %s ${quote(raw)} > /${name}.json`);
    await must("anchor", `docker cp /${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
  }

  // Assigned one at a time, because assignment resolves the whole set and says so immediately.
  // A refusal here is the graph rejecting something, which is the point of asking.
  for (const name of modules) {
    await mesh(`assign anchor ${name}`);
  }

  const plan = await mesh("plan anchor --json", 120_000);
  const declaration = JSON.parse(plan.slice(plan.indexOf("{")));
  const byId = new Map<string, any>(
    (declaration.resources as any[]).map((r) => [r.id, r]));
  const ids = [...byId.keys()];

  // Every module's own network, which only exists because more than one container needs to reach
  // another by name.
  for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) {
    assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`);
    assert.equal(byId.get(id).type, "network");
  }

  // The cross-module edge: keycloak asked for a database and was told where it is and given a
  // credential. Neither file is anything keycloak's manifest could have written.
  const bound = [...byId.values()].find((r) =>
    r.type === "file" && r.path === "/var/lib/keycloak/database.json");
  assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`);
  assert.match(JSON.stringify(bound), /postgres/,
    "keycloak's binding does not name what answered its requirement");

  // The password, alone in a file and sealed. It is a password and nothing else, so nothing reads
  // it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction.
  const credential = [...byId.values()].find((r) =>
    r.type === "file" && r.path === "/var/lib/keycloak/database.secret");
  assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`);
  assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it");
  assert.ok(!credential.content, "a credential arrived as content rather than sealed");

  // And the connection itself, which keycloak could not have written: the address and port come
  // from what the provider serves, and the user name from what the mesh decided both ends would
  // call this consumer (novox/hq 04-ISSUES/023).
  const connection = [...byId.values()].find((r) =>
    r.type === "file" && r.path === "/var/lib/keycloak/database.env");
  assert.ok(connection, "keycloak was given no database configuration");
  assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/,
    `keycloak was not told what name to present:\n${connection.content}`);
  assert.doesNotMatch(connection.content, /\$\{bound:/,
    `a placeholder reached the machine as a value:\n${connection.content}`);

  // The password is the one hole left open, and the sealed value travels beside it. The mesh
  // discarded the plaintext, so the host is the only thing that can close it.
  assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/,
    `the password was not left for the host to fill:\n${connection.content}`);
  assert.ok(connection.secrets?.["postgres-database"],
    "the sealed credential did not travel with the file that needs it");
  assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/,
    "the credential was written into the configuration in the clear");

  // And the provider was told who asked, which is what its provisioner reconciles against.
  const grants = [...byId.values()].find((r) =>
    r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants"));
  assert.ok(grants, "postgres was never told which modules were granted a database");
  assert.match(JSON.stringify(grants), /keycloak|gitea/,
    "the grants file names neither module that asked for a database");

  // Secrets reach containers as files, never as environment in the declaration.
  const containers = [...byId.values()].filter((r) => r.type === "container");
  assert.ok(containers.length >= 12,
    `only ${containers.length} containers; mailu alone is nine`);
  for (const c of containers) {
    for (const [key, value] of Object.entries(c.env ?? {})) {
      assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/,
        `${c.name} carries something secret-shaped in env.${key}, which the broker would see`);
    }
  }
});