mesh-lab/test/integration/mesh.test.ts

/**
 * A mesh, raised from nothing, joined by two machines, delivering a credential neither the mesh
 * nor the broker can read.
 *
 * Everything before this proves a part. This proves the parts meet — which is the thing the
 * project keeps saying cannot be checked any other way (novox/hq ADR 0001: every fault of
 * 2026-08-22 was found in production because nothing could be stood up locally).
 *
 * It needs a host binary and the foundation bundle:
 *
 *   MESH_LAB_HOST_BINARY=.../mesh-host
 *   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
 *
 * The bundle's image references are rewritten to the ones this scenario's own registry serves.
 * A digest belongs to whatever registry serves it, so a committed bundle names a registry that is
 * not this one — rewriting is what makes it applicable rather than a placeholder to tidy away.
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import type { HeldImage } from "../../src/pinning.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";

/** Whether this run keeps its mesh for the next one. Off unless asked for. */
const warming = process.env["MESH_LAB_WARM"] === "1";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
      // The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074).
      : catalogueIsPresent() || false;

const SCENARIO = "two-nodes";
let instanceId = "";
/**
 * The MESH's own artifact store, once the registry module is running on the anchor.
 *
 * Not a registry the lab raised — there is no longer any such thing. A build publishes into the
 * store the mesh itself runs, which is the only registry that exists outside this repository.
 */
const registry = "127.0.0.1:5000";
/**
 * The registry module's image, pinned upstream, pulled by the machine over its uplink.
 *
 * The digest mesh-catalog's `registry` module pins, so the store the mesh runs here is the store
 * the mesh runs anywhere.
 */
const ARTIFACT_STORE =
  "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];

/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
  return onTheMachine(reference, held);
}

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  // Each part on its own line, and stderr redirected once for the whole script.
  //
  // It was `${command} 2>&1; echo ...` on a single line, which quietly broke every command
  // containing a heredoc: the terminator line became `MARKER 2>&1; echo ...`, matched nothing, and
  // the heredoc swallowed the rest of the script — including the echo. `exec 2>&1` needs no
  // trailing text on the command's last line, so a heredoc terminates where it says it does.
  const { stdout } = await exec(instanceId, machine, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) {
    // **Never success.** `Number("")` is 0, so a missing marker used to read as exit 0 — a
    // command whose output was swallowed reported that it worked, which is the one answer a test
    // harness must never give.
    return { out: stdout, ok: false };
  }
  const said = stdout.slice(marker + 7).trim();
  return { out: stdout.slice(0, marker), ok: said === "0" };
}

async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(machine, command, timeoutMs);
  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
  return out;
}

/** The control plane, which runs in a container on the first node. */
async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
}

/**
 * Wait until a machine has actually applied what it was last sent.
 *
 * **`push` sends; it does not wait.** It prints "sent N resource(s)" and returns, and the machine
 * applies afterwards. So asserting on what a machine is running immediately after a push is a race
 * — and the one this fixes lost it silently: `status` still described the *previous* apply, so it
 * reported nothing wrong while the containers from this declaration did not exist yet.
 *
 * Asked of the mesh rather than of the machine, and in its own terms. A node is settled when it is
 * neither waiting for what it was sent nor wrong about what it applied — the same two questions
 * `status` answers, read as JSON so a test is not parsing a report meant for a person.
 *
 * A machine that reports a failure ends this at once rather than on the timeout: it is not going
 * to become right by being waited for, and a refusal read after two minutes of polling is the same
 * refusal, later.
 */
async function settled(node: string, withinMs = 480_000): Promise<void> {
  const until = Date.now() + withinMs;
  let last = "";
  while (Date.now() < until) {
    // **Could not ask** and **asked, and the answer was bad** are different facts, and only the
    // second is this machine's fault. The control plane is a container on the node being polled:
    // while it applies a declaration, an exec into it can lose its fifo to containerd, and a poll
    // loop that treats that as a verdict reports the mesh broken because the question missed.
    // And a poll that *threw* — an exec timeout, a lost fifo — is also "could not ask", not a
    // verdict. The distinction failed once as an IncusError surfacing at minute four of a wait
    // whose machine was merely slow.
    let state: {
      wrong: { node: string; outcome: string; refused?: string;
               failed?: { id: string; error: string }[] }[];
      waiting: { node: string; never: boolean }[];
      reported: { node: string; outcome: string; current: boolean }[];
    } | undefined;
    let said = "";
    try {
      const asked = await on("anchor",
        `docker exec mesh-controller /mesh-controller status --json`);
      said = asked.out;
      // Parsed inside the try on purpose: a truncated answer from a struggling machine is the
      // same fact as no answer, and the likeliest moment for one is exactly the machine this
      // poll is watching.
      if (asked.ok) state = JSON.parse(said);
    } catch (err) {
      said = (err as Error).message;
    }
    if (!state) {
      last = said;
      await new Promise((r) => setTimeout(r, 5000));
      continue;
    }

    const bad = state.wrong.find((w) => w.node === node);
    if (bad) {
      const why = [bad.refused, ...(bad.failed ?? []).map((f) => `${f.id}: ${f.error}`)]
        .filter(Boolean).join("\n  ");
      throw new Error(`${node} did not apply what it was sent (${bad.outcome}):\n  ${why}`);
    }
    // Caught up is an equality, not an ordering. This compared timestamps once — report newer
    // than send — and lost the race it invited: the previous test's closing apply reported
    // after this test's push, newer and still about the old declaration. The report now names
    // the declaration it applied, and the mesh says whether that is the one it last sent.
    const word = state.reported.find((r) => r.node === node);
    const acted = word?.outcome === "applied" && word.current;
    if (!state.waiting.some((w) => w.node === node) && acted) return;
    last = said;
    // Unhurried on purpose: each poll is an exec into a container on the machine that is busy
    // applying, and asking four times a minute rather than thirty is the difference between
    // observing the apply and competing with it.
    await new Promise((r) => setTimeout(r, 5000));
  }
  throw new Error(`${node} never caught up with what it was sent within ` +
    `${Math.round(withinMs / 1000)}s. Last answer, or the reason there was none:\n${last}`);
}

/**
 * The bundle, as a machine should receive it.
 *
 * The committed example was written for a target that had a registry the lab raised. Its two
 * third-party images become upstream references the machine pulls itself; mesh-controller, which
 * exists in no registry, becomes the ID this machine was handed.
 */
function bundleFor(images: HeldImage[]): string {
  return foundationBundle(bundle, images);
}

/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

/** The builder started by hand on the anchor, against the foundation broker's plain port on
 *  loopback — the one that builds until a builder module can (see the retired test's note). */
async function startBuilder(): Promise<void> {
  // The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
  // return, so it is pushed whenever the machine has none.
  if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
    await incus([
      "file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
      "--mode", "0755",
    ], 180_000);
  }
  await must("anchor", `mkdir -p /var/lib/mesh-builder`);
  await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
    `(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
    `MESH_WORKSPACE=/var/lib/mesh-builder ` +
    `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
}

before(async () => {
  if (skip) return;

  // A mesh kept between runs, when one is being kept and still counts.
  //
  // **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the
  // ones after it. Off by default: a run that is meant to mean something raises from nothing,
  // because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped
  // last week".
  if (warming) {
    const said = await ready(SCENARIO);
    if (said.use === "restore") {
      instanceId = said.instanceId;
      const seconds = await returnTo(instanceId);
      held = warmStock(instanceId).images;

      // **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
      // this suite started by hand is gone — the host most of all. Without it the mesh looks
      // perfectly healthy from the control plane's side: a module is assigned, a declaration is
      // sent and recorded, and nothing on the machine is listening to apply it. That is exactly
      // how this was first met, and it cost an hour to see.
      //
      // The real answer is a host started by init, which is what the design says it is anyway
      // (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places
      // it that way, the warm path restarts what it knows it started.
      for (const machine of ["anchor", "laptop"]) {
        await must(machine, `pgrep -x mesh-host >/dev/null || ` +
          `(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
      }
      const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
      assert.equal(running.out.trim(), "yes",
        "the host did not come back after a restore, so nothing would apply anything");
      // The hand-started builder is memory too, and the snapshot is disk.
      if (builder) await startBuilder();

      console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
        `and started the host again`);
      return;
    }
    console.log(`warm: raising fresh — ${said.why}`);
  }

  // **Progress is printed, and that is not decoration** (novox/hq 04-ISSUES/024). A raise takes
  // minutes and said nothing until it finished, so a stall and ordinary work were the same
  // thing to look at — and the one time it mattered, thirty-five minutes of nothing was read as
  // a slow test until somebody went and looked inside the machine.
  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
  });
  instanceId = raised.instanceId;

  // The first node raises everything from a file rather than from a bundle built into the binary,
  // because the control plane's image is named by the ID this machine holds it under, which is not
  // knowable until it has been handed over.
  held = raised.images;
  await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
  await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);

  // A build machine, so anything here can ask the mesh to build something. Placed rather than
  // assumed: nothing else in this scenario would start one.
  if (builder) await startBuilder();
  if (warming) {
    // Snapshotted only now, with everything up: a state worth returning to is the one after the
    // part nobody wants to repeat.
    await rememberStock(instanceId, held);
    const warm = await keep(SCENARIO, instanceId);
    console.log(`warm: ${warm.instanceId} kept, against ` +
      Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
  }
}, { timeout: 1_800_000 });

after(async () => {
  // A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything
  // else is destroyed, because an instance nobody meant to keep is one nobody will remember.
  if (warming) return;
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
  for (const container of ["mesh-store", "mesh-broker", "mesh-controller"]) {
    assert.match(running, new RegExp(container), `${container} is not running`);
  }
  // Answering, not merely up. A container that is running is not a control plane that replies —
  // a distinction this project has already paid for once.
  assert.ok((await mesh("status")).length > 0);
});

test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
  for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
    await mesh(`node add ${node}`);
    const token = tokenFrom(await mesh(`token issue --node ${node}`));
    // No --name. The token says what the mesh calls the machine, which is the fault this walk
    // found the first time it was run.
    const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
    assert.match(said, new RegExp(`enrolled as ${node}`), said);
    assert.match(said, /sealing key/, "no sealing key was generated");
  }

  const recorded = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select name from node where sealing_key is not null order by name"`);
  assert.equal(recorded.trim().split("\n").map((l) => l.trim()).sort().join(","), "anchor,laptop",
    "the mesh did not record a sealing key for both machines");
});

test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
  // The whole argument, on real machines: the two ends must hold the SAME password, and it must
  // appear nowhere the mesh or the broker could read it.
  await must("anchor", `printf %s '{"module":"a-store","version":"1",` +
    `"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
    `"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
    `"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
  // The consumer also writes a configuration file with a hole in it, which is how nearly every
  // real program takes a credential: a sealed file is a password alone, and almost nothing reads
  // one. The mesh cannot compose the document — it discarded the value — so the module supplies it
  // with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in.
  // A slug, so its identity on a backend stays within an S3 access key's 20 characters (ADR 0049).
  await must("anchor", `printf %s '{"module":"meshboard","version":"1","slug":"board",` +
    `"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
    `"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
    `"secrets":{"postgres-database":"/etc/meshboard/database.password"},` +
    `"resources":[{"id":"env","type":"file","path":"/etc/meshboard/database.env","mode":"0600",` +
    `"content":"PGHOST=$\{bound:postgres-database:at\}\\nPGPORT=$\{bound:postgres-database:port\}\\n` +
    `PGUSER=$\{bound:postgres-database:as\}\\nPGPASSWORD=$\{secret:postgres-database\}\\n"}]}' ` +
    `> /tmp/app.json`);
  await must("anchor", `docker cp /tmp/pg.json mesh-controller:/pg.json`);
  await must("anchor", `docker cp /tmp/app.json mesh-controller:/app.json`);
  await mesh("module add /pg.json");
  await mesh("module add /app.json");

  await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
  await mesh("overlay place laptop --site lab");
  for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
  await mesh("assign anchor a-store");
  await mesh("assign laptop meshboard");

  for (const machine of ["anchor", "laptop"]) {
    // Once. On a warm return the host is already running (see `before`); a second one would
    // consume the same queue and apply the same declaration twice, concurrently.
    await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
  }
  await mesh("push");
  await new Promise((r) => setTimeout(r, 8000));
  // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
  // and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
  // keeps the hub closed and nothing on the laptop reaches anchor over the private network.
  await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });

  const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
  // Named after the machine *and* the module, because a consumer is both (novox/hq
  // 04-ISSUES/022) — a node routinely runs several modules wanting one database.
  const onProvider = (await must("anchor",
    `cat /var/lib/mesh-host/grants/laptop.meshboard.secret`)).trim();
  assert.ok(onConsumer.length >= 40, `the consumer's credential is ${onConsumer.length} characters`);
  assert.equal(onConsumer, onProvider,
    "the two ends hold different passwords, so nothing could ever authenticate");

  // Only the machine it is for may read it.
  assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.password`), /^600/);

  // And the configuration with holes in it arrived filled. **This is the only place the two
  // substitutions are proven against a real host**: the mesh fills what it knows in the clear
  // before sending, the host opens the sealed value and fills the rest on the machine, and the
  // two expressions that find the holes live in different repositories.
  const filled = await must("laptop", `cat /etc/meshboard/database.env`);
  assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`);
  assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`),
    `the file holds a different password from the credential file:\n${filled}`);
  assert.match(filled, /^PGUSER=mesh_laptop_board$/m,
    `the consumer was not told what name to present:\n${filled}`);
  assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`);
  assert.doesNotMatch(filled, /\$\{/,
    `a placeholder survived to the machine and would be read as a value:\n${filled}`);
  assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.env`), /^600/);

  // The password is in that file and nowhere the mesh could read it — which is the whole point of
  // filling the hole on the machine rather than composing the document in the control plane.

  // And it is nowhere it could have been read on the way. The declaration crossed the broker; the
  // database is the control plane's; the state is what the node reported back.
  for (const [machine, where] of [
    ["laptop", "/var/lib/mesh-host/declared.json"],
    ["laptop", "/var/lib/mesh-host/state.json"],
    ["anchor", "/var/lib/mesh-host/declared.json"],
  ] as const) {
    // **`--` first, or the password is read as options.** A generated credential is random, so
    // one of them eventually begins with a dash — this one started `-S` and grep refused the
    // whole invocation. The test then compared an error message against "0" and reported the
    // password as leaked, which is the worst way for a search to fail: it says it found
    // something.
    const { out } = await on(machine, `grep -c -e ${quote(onConsumer)} -- ${where}`);
    assert.equal(out.trim(), "0", `the password is in ${where} on ${machine}`);
  }
  const inTheMesh = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select count(*) from secret where for_consumer like '%${onConsumer}%' ` +
      `or for_provider like '%${onConsumer}%'"`);
  assert.equal(inTheMesh.trim(), "0", "the control plane's database holds the password in the clear");
});

test("the consumer is also told where its database is", { skip, timeout: 300_000 }, async () => {
  // A password with no address is not a connection. This is the readable half, which stays
  // readable on purpose — it is the secret half that could not be composed, not this one.
  const told = JSON.parse(await must("laptop", `cat /etc/meshboard/database.json`));
  assert.equal(told.from, "anchor");
  assert.equal(told.at, "anchor.internal");
  assert.equal(told.serves.port, 5432);
  // And the name it resolves to was written by the mesh as well, on this machine.
  assert.match(await must("laptop", `grep anchor.internal /etc/hosts`), /10\.42\.0\.\d+/);
});

test("when a machine cannot do what it was told, the mesh says which and why", { skip, timeout: 900_000 }, async () => {
  // Demonstrated with inserted rows first, which proves the query and not the path. This sends a
  // real machine something it will genuinely fail at, and asks the mesh afterwards.
  //
  // A package that does not exist, because that is a failure of the ordinary kind: the host tries,
  // the package manager says no, and some of the declaration is applied and some is not — which
  // is the situation `status` exists to distinguish from a machine that refused everything.
  await must("anchor", `printf %s '{"module":"impossible","version":"1","resources":[` +
    `{"id":"nothing","type":"package","package":"a-package-that-does-not-exist"}]}' > /tmp/imp.json`);
  await must("anchor", `docker cp /tmp/imp.json mesh-controller:/imp.json`);
  await mesh("module add /imp.json");
  await mesh("assign laptop impossible");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 10_000));

  const said = await mesh("status");
  assert.match(said, /not doing what they were told/, said);
  assert.match(said, /laptop/, said);
  // The machine's own words about the resource that failed, not a summary written at this end.
  assert.match(said, /impossible\.nothing/, `the failing resource is not named:\n${said}`);

  // And the distinction survives: this machine FAILED, it did not refuse. Refused means it is
  // exactly as it was; failed means it is in a state nobody declared, and they are fixed in
  // different places.
  assert.match(said, /laptop\s+failed/, said);

  // The other machine is not implicated.
  assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/,
    "a machine that did as it was told is listed as wrong");
});

test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => {
  // A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so
  // a push to one that is not listening must wait rather than vanish. The queue is durable and the
  // message persistent, which ought to be enough — but a lost declaration is silent, and "ought to
  // be" is not a property.
  //
  // The machine is not merely idle here: its host is stopped, so nothing is consuming its queue.

  // Nothing this test asserts should depend on what another left behind. The machine still has a
  // deliberately-impossible module from the test above, and while that is assigned the mesh never
  // updates its account of what the machine holds — a partial report is not an account, on
  // purpose (novox/hq 04-ISSUES/010).
  await mesh("unassign laptop impossible");

  // Stop listening, and prove it stopped — a test that pushed to a machine that was still running
  // would pass having checked nothing.
  //
  // By process name, never by matching the command line: `pkill -f` matches the shell running it
  // too, which kills the connection carrying the command and hangs the caller waiting for a reply
  // that will never come. Cost an hour once, in this file.
  await must("laptop", `pkill -x mesh-host || true; sleep 1`);
  const listening = await on("laptop", `pgrep -x mesh-host`);
  assert.equal(listening.ok, false, "the host is still running, so this proves nothing");

  await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` +
    `{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`);
  await must("anchor", `docker cp /tmp/away.json mesh-controller:/away.json`);
  await mesh("module add /away.json");
  await mesh("assign laptop while-away");
  await mesh("push laptop");

  // Nothing has happened on the machine, because nothing is there to do it.
  const before = await on("laptop", `test -f /etc/mesh-while-away`);
  assert.equal(before.ok, false, "a machine with no host applied a declaration");

  // And now it listens again. No second push, and nobody says anything.
  await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`);
  let arrived = false;
  for (let i = 0; i < 20 && !arrived; i++) {
    arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok;
    if (!arrived) await new Promise((r) => setTimeout(r, 2000));
  }
  if (!arrived) {
    // Everything needed to tell "the message was never queued" from "the host never read it".
    const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
    const queues = await on("anchor",
      `docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
    const owned = await on("anchor",
      `docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
    assert.fail(`a declaration sent to a switched-off machine was lost\n` +
      `--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
      `--- what each machine last did ---\n${owned.out}`);
  }
  assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");

  // And the mesh's account of what that machine holds catches up too, or a later declaration
  // would tell it to remove what it has just been given.
  await new Promise((r) => setTimeout(r, 4000));
  const owned = await must("anchor",
    `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
      `-c "select owned from node where name = 'laptop'"`);
  assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`);
});

test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => {
  // Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares,
  // and never what the machine raised for itself from its bundle — which is the fault that
  // destroyed a foundation once (novox/hq 04-ISSUES/010).
  //
  // Two modules, so the test can tell "removed the right one" from "removed everything".
  for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) {
    await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` +
      `{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`);
    await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
    await mesh(`module add /${name}.json`);
    await mesh(`assign anchor ${name}`);
  }
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 6000));
  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive");
  assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive");

  await mesh("unassign anchor going");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 6000));

  assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false,
    "an unassigned module's file is still there");
  assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok,
    "unassigning one module took another one's file with it");

  // And the foundation this machine raised from its own bundle is untouched. It was not declared by
  // the mesh, so the mesh must never remove it — the machine would take its own control plane
  // away, which is exactly what happened before origins existed.
  const running = await must("anchor", `docker ps --format '{{.Names}}'`);
  for (const container of ["mesh-store", "mesh-broker", "mesh-controller"]) {
    assert.match(running, new RegExp(container),
      `${container} was removed by a declaration that never declared it`);
  }
});

test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => {
  // The other direction of the same rule. A node that is sent a declaration mentioning none of its
  // private network must not lose it: the network came from a module that is still assigned, and
  // "not in this message" is not "no longer wanted".
  assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok,
    "the private network's configuration is gone");
  assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok,
    "the mesh's names are gone");
});

test("a machine that fell behind catches up without being named", { skip, timeout: 900_000 }, async () => {
  // `status` says which machines are not doing what they were told; something has to act on it.
  // `push --behind` is that something, and it is a command rather than a timer to begin with —
  // a scheduler is then a scheduler over this, rather than a second path to the same act.

  // Break one machine, in a way that is fixable: a package that does not exist yet.
  await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
    `{"id":"pkg","type":"package","package":"a-package-that-does-not-exist-yet"},` +
    `{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
  await must("anchor", `docker cp /tmp/fix.json mesh-controller:/fix.json`);
  await mesh("module add /fix.json");
  await mesh("assign laptop fixable");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 8000));

  assert.match(await mesh("status"), /laptop\s+failed/, "the machine did not report a failure");
  // And the resources that COULD be applied were — one broken thing no longer blocks the rest
  // (novox/hq 04-ISSUES/011).
  assert.ok((await on("laptop", `test -f /etc/mesh-fixable`)).ok,
    "a resource after the failing one was never attempted");

  // Nothing is behind on the other machine, so nothing is pushed to it.
  const named = await mesh("push --behind");
  assert.match(named, /laptop/, named);
  assert.doesNotMatch(named, /sent anchor/, `a machine that was fine was pushed to:\n${named}`);

  // Fix the cause, the way somebody would: the module stops asking for the impossible thing.
  await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
    `{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
  await must("anchor", `docker cp /tmp/fix.json mesh-controller:/fix.json`);
  await mesh("module add /fix.json");

  // And nobody names the machine.
  await mesh("push --behind");
  await new Promise((r) => setTimeout(r, 8000));

  const after = await mesh("status");
  assert.doesNotMatch(after, /laptop\s+(failed|refused)/,
    `the machine did not recover:\n${after}`);

  // With nothing behind, it says so rather than doing nothing quietly.
  assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
});

test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async () => {
  // Artifacts go to a registry, and a mesh had no way to run its own — the only one that existed
  // was raised by the lab, which is to say it existed nowhere but here.
  //
  // **Named, not mirrored** (novox/hq 04-ISSUES/029). Mirroring publishes to the artifact store,
  // and the builder will not start without one — so a module that provides the store and builds
  // its own image asks the mesh to put an artifact into the thing that artifact is needed to
  // create. It used to pass here only because the LAB's registry was already standing to receive
  // the push, which is exactly why a real first mesh would have found this and the lab did not.
  //
  // So the image is named by digest, the way the bundle names the three a first node starts from.
  // A registry is the one module that cannot be delivered by the mesh's own delivery.
  await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
    `"provides":[{"name":"artifact-store","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"claims":[{"name":"the-artifact-store","scope":"node"}],` +
    `"serves":{"artifact-store":{"port":5000}},` +
    `"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
    `"resources":[` +
    `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
    `{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
    `"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
    `> /root/registry/module.json`);
  // **Added, not built** — and this is the half that proves the fix. Building needs a builder,
  // and a builder will not start without an artifact store to publish to, so a mesh that has just
  // bootstrapped cannot build the module that gives it one. Adding the manifest directly is the
  // path a real first mesh has to take, so it is the path this walks.
  await must("anchor", `docker cp /root/registry/module.json mesh-controller:/registry.json`);
  await mesh("module add /registry.json");
  await mesh("assign anchor registry");
  await mesh("push anchor");
  // The store's image is pulled from upstream at apply, over the uplink; that takes what it takes.
  let names = "";
  for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
    await new Promise((r) => setTimeout(r, 3000));
    names = await must("anchor", `docker ps --format '{{.Names}}'`);
  }
  // Running, and answering — a container that is up is not a registry that replies.
  assert.match(names, /mesh-registry/,
    `the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
  let answers = false;
  for (let i = 0; i < 20 && !answers; i++) {
    answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
    if (!answers) await new Promise((r) => setTimeout(r, 2000));
  }
  assert.ok(answers, "the mesh's own registry is running and does not answer");

  // And reachable from another machine over the private network, which is the whole point of an
  // artifact store being a mesh-scoped provision.
  assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
    "the artifact store is not reachable from another machine, so nothing else can use it");
});

// Defends novox/hq ADR 0007: the mesh is its own certificate authority for internal names.
test("a machine serves its internal name with a certificate the mesh issued", {
  skip, timeout: 900_000,
}, async () => {
  // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
  // Asserted with a real handshake: a certificate that parses and does not chain fails at the
  // moment something connects, which is the worst place to find out.
  // The port the handshake below is tried on, declared: the anchor filters what its modules
  // did not declare (ADR 0088), and a test server on an undeclared port proves only that.
  await must("anchor", `printf %s '{"module":"served","version":"1",` +
    `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
    `"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
    `> /tmp/served.json`);
  await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
  await mesh("module add /served.json");
  await mesh("assign anchor served");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 8000));

  assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
  assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");

  // The name it was issued for is the one the mesh gave this machine.
  const named = await must("anchor",
    `openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
    `docker run --rm -v /etc/mesh:/m ${ARTIFACT_STORE} sh -c ` +
    `"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
  assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);

  // And a real handshake: the machine serves TLS with the key it generated, and another machine
  // verifies it against the mesh's authority and nothing else.
  await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
    `-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
    `> /var/log/tls.log 2>&1 & sleep 2`);
  await must("laptop", `mkdir -p /etc/mesh`);
  const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
  await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);

  const shook = await on("laptop",
    `echo | openssl s_client -connect anchor.internal:8443 ` +
    `-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
  assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` +
    `what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`);
  assert.match(shook.out, /Verification: OK/, shook.out);
});

// Defends novox/hq ADR 0007: what a machine exposes is what its modules declared, and nothing
// arrives at a port nobody asked for.
test("a machine filters exactly what its modules declared, and nothing else", {
  skip, timeout: 900_000,
}, async () => {
  // The rule set is derived from what is assigned, not kept in step by hand — and the proof that
  // matters is not that a file arrived but that packets are treated differently because of it.
  // A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
  //
  // Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
  // so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
  // the shape that rule leaves, and this is the first thing to use it for its real purpose.
  // A listener is written to a file rather than squeezed through three levels of shell quoting.
  // The first attempt did the latter, never started, and the test failed on its own setup —
  // which reads exactly like the firewall working.
  await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` +
    `import socket, sys, threading\n` +
    `def serve(port):\n` +
    `    s = socket.socket()\n` +
    `    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` +
    `    s.bind(("0.0.0.0", port))\n` +
    `    s.listen(8)\n` +
    `    while True:\n` +
    `        c, _ = s.accept()\n` +
    `        c.send(str(port).encode())\n` +
    `        c.close()\n` +
    `for p in (9101, 9102):\n` +
    `    threading.Thread(target=serve, args=(p,), daemon=True).start()\n` +
    `threading.Event().wait()\n` +
    `LISTENER`);
  await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`);

  // Two paths to the same machine, which is what makes "from the mesh" testable at all: over the
  // private network, and over the segment both machines happen to share. A rule that opens a port
  // to the mesh must accept the first and refuse the second — and a test that only ever used one
  // path could not tell "open to the mesh" from "open".
  const reach = async (where: string, port: number) => {
    const said = await on("anchor",
      `timeout 5 python3 -c "import socket;s=socket.create_connection(('${where}',${port}),4);` +
      `print(s.recv(32).decode());s.close()"`);
    return said.ok;
  };
  const overlay = (port: number) => reach("laptop.internal", port);
  const segment = (port: number) => reach("192.0.2.20", port);

  // Reachable both ways before any rule set exists, so what changes afterwards is the rule set and
  // not the listener. Without this the test would pass against a service that never started.
  assert.ok(await overlay(9101), "the declared port never opened, so nothing below tests anything");
  assert.ok(await overlay(9102), "the undeclared port never opened");
  assert.ok(await segment(9101), "the declared port is not reachable off the private network yet, " +
    "so closing it later would prove nothing");

  await must("anchor", `printf %s '{"module":"talker","version":"1",` +
    `"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
    `"resources":[]}' > /tmp/talker.json`);
  // The rule set goes where this machine's nftables unit reads from, and the unit is declared to
  // reflect it. No command anywhere.
  // The module ships the unit that loads its rules, rather than using the one the distribution's
  // nftables package provides. That unit is `Type=oneshot` with no `RemainAfterExit`, so it does
  // its work and goes inactive — and a host asked for a service that is "running" reports, quite
  // correctly, that it is stopped. There is no state in the vocabulary for "ran and exited having
  // done its job", so a module that wants one brings a unit that stays.
  //
  // Which is also the right shape: how a machine enforces rules is a fact about the machine, and
  // the mesh has no business depending on what a distribution happens to package.
  await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
    `"capabilities":["firewall"],` +
    `"filtering":{"into":"/etc/mesh/filter.nft"},` +
    `"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
    `{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"},` +
    `{"id":"unit","type":"file","path":"/etc/systemd/system/mesh-filter.service",` +
    `"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for this machine\\n` +
    `[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
    `ExecStart=/usr/bin/nft -f /etc/mesh/filter.nft\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
    `{"id":"filter","type":"service","unit":"mesh-filter.service","state":"running",` +
    `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
  for (const f of ["talker", "firewall"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign laptop talker");
  await mesh("assign laptop firewall");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 20_000));

  const written = await must("laptop", `cat /etc/mesh/filter.nft`);
  // A rule names its source. Not decoration: it is the only thing that answers "why is this open".
  assert.match(written, /# talker . the thing this test is about/,
    `the rule does not name what caused it:\n${written}`);
  // The mesh's addresses are the ones on the private network, which is what "from the mesh"
  // means — not the segment the machines happen to share.
  assert.match(written, /ip saddr \{ [0-9., ]+ \} tcp dport 9101 accept/,
    `"from the mesh" resolved to nothing:\n${written}`);
  assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);

  // Loaded, not merely written. The service was restarted because a file it reflects changed.
  const table = await must("laptop", `nft list table inet mesh`);
  assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);

  // And it filters. Three assertions, and the third is the one that makes "from the mesh" mean
  // something rather than being a synonym for "open".
  assert.ok(await overlay(9101),
    "the declared port is closed on the private network, so the machine is filtering more than " +
    "it was told to");
  assert.ok(!(await overlay(9102)),
    "a port no module declared is still reachable, so the rule set restricts nothing");
  assert.ok(!(await segment(9101)),
    "the declared port answers off the private network, so `from: mesh` restricted nothing");

  // The machine did not lock itself out of the mesh: it is still taking declarations.
  assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
    "the machine stopped doing what it was told after applying its own rule set");

  // Removing the module that wanted the port closes it, with nobody editing a rule. This is the
  // whole claim of a derived firewall, and it is also the second load — which must replace the
  // table rather than add to it.
  await mesh("unassign laptop talker");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 20_000));
  assert.ok(!(await overlay(9101)),
    "the port stayed open after the module that wanted it was removed");
});

// The builder as a module the mesh assigns, with a credential the mesh delivered, is what genesis
// proves now (genesis-single installs the catalogue's builder through the installer, novox/hq ADR
// 0069). The test that lived here declared the builder's image as an upstream artifact by the bare
// image ID the lab holds, which is not a reference a registry copy can fetch (ADR 0096); retired
// 2026-09-21 rather than rewritten into a second genesis.

test("rotating a credential moves both ends, and the old one stops working", {
  skip, timeout: 900_000,
}, async () => {
  // The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: on
  // 2026-08-22 a provision documented as never rotating minted a new password on every adoption
  // and updated only the provider's row. Consumers on three nodes held dead credentials for two
  // days while the mesh reported success.
  //
  // So this is checked against a real database with a real login, three times: the delivered
  // credential works, the rotated one works, and the one that was rotated away does not. Two ends
  // holding a matching string proves they agree; only an authentication proves they are right.
  const store = "/var/lib/mesh/postgres";
  await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
    `"provides":[{"name":"real-postgres-database","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"serves":{"real-postgres-database":{"port":5433}},` +
    `"own-secrets":{"superuser":"${store}/superuser"},` +
    `"grants":{"real-postgres-database":"${store}/grants"},` +
    // Both halves. `grants` is where each consumer's sealed password lands; `receives` is the
    // manifest saying who asked and for what. Without the second the provisioner finds a
    // directory of unexplained secrets and says nothing has been granted — which is true, and
    // reads exactly like a credential that was never delivered.
    `"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` +
    `"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
    `"resources":[` +
    `{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
    `{"id":"grants","type":"directory","path":"${store}/grants","mode":"0755"},` +
    `{"id":"postgres-database","type":"container","name":"real-store",` +
    `"image":"${pinned("postgres")}",` +
    `"ports":["5433:5432"],` +
    `"volumes":["${store}/superuser:/run/superuser:ro"],` +
    `"env":{"POSTGRES_PASSWORD_FILE":"/run/superuser"}},` +
    `{"id":"provisioner","type":"container","name":"real-provisioner",` +
    `"image":"${pinned("mesh-provision-postgres")}","network":"host",` +
    `"volumes":["${store}:${store}:ro"],` +
    `"env":{"GRANTS":"${store}/grants",` +
    `"MESH_PROVISION_PASSWORD_FILE":"${store}/superuser",` +
    `"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
    `> /tmp/realstore.json`);
  await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
    `"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` +
    `"binds":{"real-postgres-database":"/etc/realapp/where.json"},` +
    `"secrets":{"real-postgres-database":"/etc/realapp/password"},` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
    `> /tmp/realapp.json`);
  for (const f of ["realstore", "realapp"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign anchor realstore");
  await mesh("assign laptop realapp");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 30_000));

  // A real login from the consumer's machine, over the private network — not over loopback, where
  // pg_hba trusts anything and every password looks correct. That was done here once and the test
  // passed for an afternoon while verifying nothing: a deliberately wrong password returned a row.
  // As the role the provisioner made, into the database it made. The provisioner names a role
  // after the machine and a database after what the module asked for — which is the contract, and
  // getting it wrong here made the test fail against a provisioner that had done its job.
  // By address, resolved on the machine.
  //
  // **This container is not the mesh's.** The mesh gives its names to the containers it declares,
  // and this one is started by the test with `docker run` — nothing declared it, so nothing
  // configured it. That boundary is the right one: a container somebody runs by hand is not the
  // mesh's to configure, and reaching into every container on a machine is what a resolver in
  // resolv.conf would be for.
  //
  // So the workaround stays here, and the proof that names work inside containers is its own
  // test, against a container the mesh declared.
  const where = (await must("laptop",
    `getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
  assert.match(where, /^[0-9.]+$/, `the mesh's name for anchor does not resolve here: ${where}`);

  const login = async (password: string) =>
    await on("laptop", `docker run --rm -e PGPASSWORD=${quote(password)} ` +
      // The role the provisioner made: mesh_<node>_<module>, because a consumer is a module on
      // a machine (novox/hq 04-ISSUES/022).
      `${pinned("postgres")} psql -h ${where} -p 5433 -U mesh_laptop_realapp ` +
      `-d realapp -qAt -c "select 1"`, 120_000);

  const diagnostics = async () =>
    `provisioner:\n${(await on("anchor", `docker logs real-provisioner 2>&1 | tail -20`)).out}\n` +
    `grants:\n${(await on("anchor", `ls -l ${store}/grants`)).out}`;

  const first = (await must("laptop", `cat /etc/realapp/password`)).trim();
  assert.ok(first.length >= 40, `the consumer's credential is ${first.length} characters`);
  let works = false;
  for (let i = 0; i < 20 && !works; i++) {
    works = (await login(first)).ok;
    if (!works) await new Promise((r) => setTimeout(r, 5000));
  }
  assert.ok(works, `the delivered credential does not authenticate:\n` +
    `${(await login(first)).out}\n${await diagnostics()}`);

  // Now rotate. One command: the record changes AND both ends are sent, because leaving the
  // sending to a later command is the fault above, exactly.
  const said = await mesh("rotate real-postgres-database", 180_000);
  assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
  assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
  await new Promise((r) => setTimeout(r, 25_000));

  const second = (await must("laptop", `cat /etc/realapp/password`)).trim();
  assert.notEqual(second, first, "the consumer was handed back the credential just rotated away");

  // The new one authenticates — the only proof the provider was told the same thing the consumer
  // was given. Two files agreeing proves they agree, not that either is right.
  let now = false;
  for (let i = 0; i < 20 && !now; i++) {
    now = (await login(second)).ok;
    if (!now) await new Promise((r) => setTimeout(r, 5000));
  }
  assert.ok(now, `after rotation the new credential does not authenticate, so the two ends ` +
    `disagree — which is the fault this exists to make impossible:\n${await diagnostics()}`);

  // And the old one does not. Without this the test passes against a provider that added a
  // password without replacing one, which is a rotation that rotates nothing.
  assert.ok(!(await login(first)).ok,
    "the password that was rotated away still authenticates, so nothing was rotated");
});

test("a route is a grant: a workload is reached by the name it asked for", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq 08-connectivity §3. The mirror of a database grant: there the consumer supplies a
  // name and receives credentials; here it supplies a target and receives a name. Nothing new in
  // the vocabulary — a route is a provision like any other.
  //
  // The workload is the registry image, because it is an HTTP server this scenario already has.
  // What is being tested is the mesh's arrangement, not the workload.
  await must("anchor", `printf %s '{"module":"frontdoor","version":"1",` +
    `"provides":[{"name":"route","scope":"mesh"}],` +
    `"capabilities":["container-runtime"],` +
    `"receives":{"route":"/etc/frontdoor/routes.json"},` +
    `"serves":{"route":{"domain":"mesh.test"}},` +
    `"listens":[{"port":8081,"from":"mesh","why":"the front door"}],` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/frontdoor","mode":"0755"},` +
    `{"id":"proxy","type":"container","name":"front-door",` +
    `"image":"${pinned("mesh-route-proxy")}","network":"host",` +
    `"volumes":["/etc/frontdoor:/etc/frontdoor:ro"],` +
    `"env":{"ROUTES":"/etc/frontdoor/routes.json","LISTEN":":8081"}}]}' ` +
    `> /tmp/frontdoor.json`);
  // The workload declares the port it listens on as well as the route it wants. Both, because
  // they are different questions: one says who may reach it, the other says by what name — and
  // the earlier test left this machine filtering, so a module that asked for a route and not for
  // the port would be unreachable by the proxy it just asked for.
  await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` +
    `"requires":["route"],"capabilities":["container-runtime"],` +
    `"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
    `"binds":{"route":"/etc/storefront/route.json"},` +
    `"listens":[{"port":8088,"from":"mesh","why":"the proxy reaches it here"}],` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/storefront","mode":"0755"},` +
    `{"id":"app","type":"container","name":"storefront",` +
    `"image":"${ARTIFACT_STORE}","ports":["8088:5000"]}]}' > /tmp/storefront.json`);
  for (const f of ["frontdoor", "storefront"]) {
    await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
    await mesh(`module add /${f}.json`);
  }
  await mesh("assign anchor frontdoor");
  await mesh("assign laptop storefront");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 25_000));

  // The provider was told who asked, and where that machine is — which it needs in order to
  // reach back, and which it must not have to derive from a naming convention.
  const routes = await must("anchor", `cat /etc/frontdoor/routes.json`);
  assert.match(routes, /shop\.mesh\.test/, `the proxy was not told about the route:\n${routes}`);
  assert.match(routes, /"at": *"laptop\.internal"/,
    `the proxy was not told where the consumer is, so it cannot reach it:\n${routes}`);

  // And the consumer was told what the provider serves, which is how it knows its own name.
  const bound = await must("laptop", `cat /etc/storefront/route.json`);
  assert.match(bound, /mesh\.test/, `the consumer was not told the public name:\n${bound}`);

  // The whole point: a request for the name reaches the workload, across the private network.
  let reached = false;
  let said = "";
  for (let i = 0; i < 20 && !reached; i++) {
    const answer = await on("anchor",
      `curl -sf -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
    said = answer.out;
    reached = answer.ok && said.trim() === "200";
    if (!reached) await new Promise((r) => setTimeout(r, 4000));
  }
  assert.ok(reached, `a request for the name did not reach the workload (${said}):\n` +
    `${(await on("anchor", `docker logs front-door 2>&1 | tail -20`)).out}`);

  // Withdrawal, which 08-connectivity lists as open: a stale public name pointing at nothing
  // fails more visibly than a stale grant, so it must not survive the module leaving.
  await mesh("unassign laptop storefront");
  await mesh("push");

  // **Waited for, not slept through.** The mesh withdrawing a route and the machine acting on it
  // are different things, and a fixed sleep between them tests whichever the clock happened to
  // land on — this assertion passed twice and failed once on nothing but timing.
  //
  // A machine that has not applied yet is also not a machine that failed, so `status` cannot
  // stand in for this: "not yet" and "never" look identical there, and only one of them is worth
  // failing over.
  let after = "";
  let withdrawn = false;
  for (let i = 0; i < 20 && !withdrawn; i++) {
    after = await must("anchor", `cat /etc/frontdoor/routes.json`);
    withdrawn = !after.includes("shop.mesh.test");
    if (!withdrawn) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(withdrawn,
    `the route outlived the module that asked for it:\n${after}\n\n` +
    `the machine did apply — this is what the mesh would send now:\n` +
    `${(await on("anchor", `docker exec mesh-controller /mesh-controller plan anchor --files`)).out}`);

  let gone = false;
  for (let i = 0; i < 15 && !gone; i++) {
    const answer = await on("anchor",
      `curl -s -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
    gone = answer.out.trim() === "404";
    if (!gone) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(gone, "the proxy still serves a name whose module was unassigned");
});

test("model access is answered by a record, and the key the mesh took is one it cannot read", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq ADR 0024. The first provision no machine answers: a hosted model is on nobody's
  // node and is reached over the public internet, so the rule that refuses two ends sharing no
  // private network must not apply to it.
  await must("anchor", `printf %s '{"module":"assistant","version":"1",` +
    `"requires":["model-access"],` +
    `"binds":{"model-access":"/etc/assistant/model.json"},` +
    `"secrets":{"model-access":"/etc/assistant/key"},` +
    `"resources":[{"id":"dir","type":"directory","path":"/etc/assistant","mode":"0755"}]}' ` +
    `> /tmp/assistant.json`);
  await must("anchor", `docker cp /tmp/assistant.json mesh-controller:/assistant.json`);
  await mesh("module add /assistant.json");

  // The licences first. With none recorded at all the honest answer is that nothing provides
  // model-access — correct, and a different refusal from the one being tested.
  await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`);
  await mesh(`licence add anthropic the-organisation --serves '{"model":"a-model"}'`);

  // Refused at the earliest point somebody could meet it: assigning records the assignment and
  // then says the machine's set cannot be applied. The refusal names both candidates and the
  // command. ADR 0024 warns this will be felt — which is correct, and correct is not the same as
  // usable.
  const refused = await on("anchor", `docker exec mesh-controller /mesh-controller assign laptop assistant`);
  assert.ok(!refused.ok,
    `a consumer was given model access without anybody saying which:\n${refused.out}`);
  for (const want of ["personal", "the-organisation", "licence use"]) {
    assert.match(refused.out, new RegExp(want),
      `the refusal does not name ${want}:\n${refused.out}`);
  }

  await mesh("licence use personal laptop assistant");

  // Chosen, and still no key: the mesh has one thing to deliver and has not been given it.
  const noKey = await on("anchor", `docker exec mesh-controller /mesh-controller plan laptop`);
  assert.ok(!noKey.ok, `a module was planned with a licence that has no key:\n${noKey.out}`);
  assert.match(noKey.out, /licence key personal/, noKey.out);

  // The accept verb. Given on standard input rather than as an argument, because a key in a
  // command line is a key in shell history and in every process listing taken while it ran.
  const secret = "sk-test-" + "0123456789abcdef".repeat(2);
  const accepted = await must("anchor",
    `printf %s ${quote(secret)} | docker exec -i mesh-controller /mesh-controller licence key personal`);
  assert.match(accepted, /sealed to 1 holder/, accepted);
  assert.doesNotMatch(accepted, new RegExp(secret),
    "the key was echoed back, so the one copy that matters is on a terminal");

  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));

  // What is public arrives, and says it is a record rather than leaving an empty address that a
  // reader would take for something the mesh failed to fill in.
  const bound = await must("laptop", `cat /etc/assistant/model.json`);
  assert.match(bound, /personal/, `the consumer was not told which licence it is on:\n${bound}`);
  assert.match(bound, /a-model/, `what the licence serves did not arrive:\n${bound}`);
  assert.match(bound, /not a machine/, `the binding leaves an unexplained empty address:\n${bound}`);

  // And the key arrives, readable only by this machine.
  assert.equal((await must("laptop", `cat /etc/assistant/key`)).trim(), secret,
    "the key that arrived is not the key that was given");
  assert.match(await must("laptop", `stat -c %a /etc/assistant/key`), /^600/);

  // The mesh cannot read it back. This is the whole argument: what is stored is unusable by
  // whoever holds it, the control plane included.
  const stored = await must("anchor",
    `docker exec mesh-store psql -U postgres -d licences -qAt ` +
    `-c "select coalesce(sealed,'') from licence_holder"`);
  assert.ok(stored.trim().length > 0, "nothing was stored, so nothing was sealed");
  assert.doesNotMatch(stored, new RegExp(secret),
    "the key is in the control plane's own database in the open");

  // Nor is it anywhere it could have been read on the way.
  for (const where of ["/var/lib/mesh-host/declared.json", "/var/lib/mesh-host/state.json"]) {
    // Whether grep found it, not how many times. `grep -c` prints 0 and exits non-zero when it
    // finds nothing, so the obvious `|| echo 0` prints a second one and the count is never what
    // it looks like.
    const found = await on("laptop", `grep -q ${quote(secret)} ${where}`);
    assert.ok(!found.ok, `the key is in the open in ${where}`);
  }
});

test("a new commit reaches a machine that is already running the old one", {
  skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
  timeout: 900_000,
}, async () => {
  // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
  // question "did my change go out?". This is that question, end to end — a commit, a build, a
  // catalogue, and a machine that ends up running what the source says.
  const repo = "/var/lib/mesh/builder/repositories/delivered";
  const write = async (what: string) =>
    await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
      `"resources":[{"id":"marker","type":"file","path":"/etc/delivered",` +
      `"content":"${what}","mode":"0644"}]}' > ${repo}/module.json`);

  await write("first");
  await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm first`);
  await mesh(`build ${repo} --wait 300s`, 420_000);
  await mesh("assign laptop delivered");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first");

  // Nothing has moved, so nothing is behind — and it says so rather than doing nothing quietly.
  assert.match(await mesh("build --behind"), /every module the mesh holds is what its source last had/);

  // Now the source moves.
  await write("second");
  await must("anchor", `cd ${repo} && git add -A && ` +
    `git -c user.email=lab -c user.name=lab commit -qm second`);
  const moved = (await must("anchor", `cd ${repo} && git rev-parse HEAD`)).trim();
  await mesh(`module moved delivered ${moved}`);

  // The mesh says which module is behind, and by how much, before anything is built.
  const behind = await mesh("status");
  assert.match(behind, /delivered/, `status does not name the module that moved:\n${behind}`);
  assert.match(behind, /build --behind/, `status does not say how to catch up:\n${behind}`);

  // The loop, in one command: everything behind its source is built and recorded.
  const built = await mesh("build --behind --wait 300s", 420_000);
  assert.match(built, /delivered/, built);

  // The machine is still running the old one until it is told — the mesh changing its mind is
  // not the same as a machine acting on it, and collapsing the two is how a mesh reports success
  // for something that has not happened.
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first",
    "the machine changed before anything was sent to it");

  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));
  assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "second",
    "the machine is still running what the source no longer says");

  // And it is no longer out of date, which is the half that makes the answer trustworthy: a
  // status that says "behind" for ever is one nobody reads.
  const after = await mesh("status");
  assert.doesNotMatch(after, /delivered.*<.*[0-9a-f]{8}/,
    `the module is still reported as behind after catching up:\n${after}`);
});

test("the board names the machine that is not doing what it was told", {
  skip, timeout: 900_000,
}, async () => {
  // novox/hq 03-DESIGN/01-to-be/11-a-board.md. The board being replaced reads every context's
  // database directly; this one asks the same questions through the same functions the commands
  // use, and holds nothing. So the check is that what it says matches what the mesh says, and
  // that it says the thing a person opened it for.
  await must("anchor", `printf %s '{"module":"board","version":"1",` +
    `"listens":[{"port":8090,"from":"mesh","why":"the board"}],` +
    `"resources":[]}' > /tmp/board.json`);
  await must("anchor", `docker cp /tmp/board.json mesh-controller:/board.json`);
  await mesh("module add /board.json");

  // Served from the control plane's own container, reading the mesh on every request.
  await must("anchor", `docker exec -d mesh-controller /mesh-controller board --listen 0.0.0.0:8090`);
  await new Promise((r) => setTimeout(r, 3000));

  const read = async (path: string) =>
    await on("anchor", `curl -sf http://127.0.0.1:8090${path}`, 30_000);

  let up = false;
  let said = { out: "", ok: false };
  for (let i = 0; i < 15 && !up; i++) {
    said = await read("/");
    up = said.ok;
    if (!up) await new Promise((r) => setTimeout(r, 2000));
  }
  assert.ok(up, `the board does not answer:\n${said.out}`);

  // Something a person opened it for: give a machine a declaration it cannot apply.
  //
  // A unit that does not exist, because the host says so immediately and in its own words. A file
  // in a missing directory is not impossible — the host creates the parents, which is correct and
  // made the first version of this test break nothing at all.
  await must("anchor", `printf %s '{"module":"impossible","version":"1",` +
    `"resources":[{"id":"nowhere","type":"service","unit":"nothing-like-this.service",` +
    `"state":"running"}]}' > /tmp/impossible.json`);
  await must("anchor", `docker cp /tmp/impossible.json mesh-controller:/impossible.json`);
  await mesh("module add /impossible.json");
  await mesh("assign laptop impossible");
  await mesh("push laptop");

  let named = false;
  let page = "";
  for (let i = 0; i < 20 && !named; i++) {
    page = (await read("/")).out;
    named = page.includes("laptop") && page.includes(">failed<");
    if (!named) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(named, `the board does not name the machine that failed:\n${page}`);

  // The host's own words, which say exactly what it could not do. A board that said only
  // "failed" would make a person go and ask the thing they opened the board to avoid asking.
  assert.match(page, /nowhere/, `the board does not say what failed:\n${page}`);

  // It agrees with the command, because both read the same thing. Two answers to "which machine
  // is broken" is worse than either alone.
  const asJSON = JSON.parse((await read("/mesh.json")).out);
  assert.equal(asJSON.wrong[0].node, "laptop", `the page and the JSON disagree: ${JSON.stringify(asJSON)}`);
  assert.equal(asJSON.wrong[0].outcome, "failed");

  const fromCommand = JSON.parse(await mesh("status --json"));
  assert.deepEqual(asJSON.wrong, fromCommand.wrong,
    "the board and the command disagree about which machine is broken");

  // And it changes nothing: the mesh is exactly as it was after being read.
  const before = await mesh("status");
  await read("/");
  assert.equal(await mesh("status"), before, "reading the board changed the mesh");

  await mesh("unassign laptop impossible");
  await mesh("push laptop");
});

// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a
// filter module derives the rules — so the credential test above assigns the catalogue's and
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh
// was not severed. Retired 2026-09-22.

test("a container reaches another machine by the name the mesh gave it", {
  skip, timeout: 900_000,
}, async () => {
  // Internal names are written to the machine's hosts file, which serves the machine and not what
  // the machine runs: a container gets its own hosts file holding only its own hostname. So every
  // name the mesh wrote was invisible to the majority of things that need one.
  //
  // Checked from inside a container rather than on the machine, because on the machine it has
  // always worked — and that is exactly what made this easy to miss.
  await must("anchor", `printf %s '{"module":"resolves","version":"1",` +
    `"capabilities":["container-runtime"],` +
    `"resources":[{"id":"idle","type":"container","name":"resolves",` +
    `"image":"${ARTIFACT_STORE}"}]}' > /tmp/resolves.json`);
  await must("anchor", `docker cp /tmp/resolves.json mesh-controller:/resolves.json`);
  await mesh("module add /resolves.json");
  await mesh("assign laptop resolves");
  await mesh("push laptop");
  await new Promise((r) => setTimeout(r, 15_000));

  // The names are in the container's own hosts file, written by the runtime.
  const inside = await must("laptop", `docker exec resolves cat /etc/hosts`);
  assert.match(inside, /anchor\.internal/,
    `the container cannot see the other machine's name:\n${inside}`);
  assert.match(inside, /laptop\.internal/,
    `the container cannot see its own machine's name:\n${inside}`);

  // And the name actually reaches the machine, which is the part that matters: a hosts entry
  // pointing at the wrong address resolves perfectly and connects to nothing.
  const reached = await on("laptop",
    `docker exec resolves sh -c 'getent hosts anchor.internal'`);
  assert.ok(reached.ok, `the name does not resolve inside the container:\n${reached.out}`);
  const address = reached.out.trim().split(/\s+/)[0];
  const onTheMachine = (await must("laptop",
    `getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
  assert.equal(address, onTheMachine,
    "the container and its machine disagree about where the other machine is");

  await mesh("unassign laptop resolves");
  await mesh("push laptop");
});

// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
// told each one.
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
 *  it is issued once per machine. The mesh writes the resolver's data as a fact the module
 *  declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
const resolverIssued = new Set<string>();
async function resolverModules(machines: string[]): Promise<void> {
  for (const name of ["dnsmasq", "resolved-split-dns"]) {
    const manifest = catalogueModule(name, held);
    await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
    await mesh(`module add /${name}.json`);
  }
  for (const machine of machines) {
    if (resolverIssued.has(machine)) continue;
    await mesh(`module issue dnsmasq --node ${machine}`);
    resolverIssued.add(machine);
  }
}

test("every name under a machine resolves to that machine", {
  skip, timeout: 900_000,
}, async () => {
  // Services are named under the machine they run on — postgres.novox.internal,
  // plex.ace.internal. The first label is the service and the rest is the node, so what must
  // resolve is anything under a node's name. What routes it once it arrives is a proxy's, and
  // stays separate.
  //
  // The mesh writes the data and runs no daemon: a resolver is third-party software, and the
  // mesh has no business choosing one. So what is checked here is the mesh's half — that the
  // data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq
  // declares, so that module is what is assigned; what it runs is the next test's concern.
  await resolverModules(["anchor", "laptop"]);
  await mesh("assign anchor dnsmasq");
  await mesh("assign laptop dnsmasq");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 15_000));

  for (const machine of ["anchor", "laptop"]) {
    const written = await must(machine, `cat /etc/mesh-resolver/nodes.conf`);

    // A wildcard per machine, matching the name and everything under it. Both machines get the
    // whole mesh: a node resolves every other node, and itself.
    for (const node of ["anchor", "laptop"]) {
      assert.match(written, new RegExp(`address=/${node}\\.internal/10\\.42\\.0\\.\\d+`),
        `${machine} cannot resolve names under ${node}:\n${written}`);
    }

    // And the addresses agree with what the machine's own hosts file says. Two accounts of where
    // a machine is, disagreeing, would be worse than either alone — and this is the one place
    // they could drift, because they are generated separately.
    const hosts = await must(machine, `getent hosts anchor.internal | head -1 | cut -d' ' -f1`);
    assert.match(written, new RegExp(`address=/anchor\\.internal/${hosts.trim().replace(/\./g, "\\.")}`),
      `the resolver data and the hosts file disagree about where anchor is:\n${written}`);
  }

  // It follows the machines. A node leaving the private network must stop being answered for,
  // because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
  // fails at once and says which name it was.
  //
  // Both: the resolver's data follows the private network, so the machine leaves the network as
  // well as the resolver, and what is asserted is that the machine that stayed is answered for and
  // the one that left is not.
  await mesh("unassign laptop dnsmasq");
  await mesh("unassign laptop networking");
  await mesh("push anchor");
  await new Promise((r) => setTimeout(r, 15_000));

  const after = await must("anchor", `cat /etc/mesh-resolver/nodes.conf`);
  assert.doesNotMatch(after, /address=\/laptop\.internal\//,
    `a machine that left the private network is still answered for:\n${after}`);
  assert.match(after, /address=\/anchor\.internal\//,
    `the machine that stayed lost its own name:\n${after}`);

  await mesh("assign laptop networking");
  await mesh("unassign anchor dnsmasq");
  await mesh("push");
  await new Promise((r) => setTimeout(r, 15_000));
});

test("a service is reached by a name under the machine it runs on", {
  skip, timeout: 900_000,
}, async () => {
  // postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
  // the node, so anything under a node's name must resolve to that node. What routes it once it
  // arrives is a proxy's concern and stays separate.
  //
  // The mesh writes the data; a module runs the daemon. Both manifests are read from the
  // repository rather than written here, so what is proven is what ships.
  // `resolved-split-dns`, not `resolv-conf`: these machines run systemd-resolved, which owns
  // /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
  // refusal rather than a fight over the file — and picking the wrong one here would have been
  // testing that fight.
  await resolverModules(["anchor", "laptop"]);

  // Both machines, because a node resolves from its own copy — the same rule as everything else
  // it holds. A mesh where one machine answers for all of them stops resolving when that machine
  // does, which is the arrangement this design refuses everywhere else.
  for (const machine of ["anchor", "laptop"]) {
    await mesh(`assign ${machine} dnsmasq`);
    await mesh(`assign ${machine} resolved-split-dns`);
  }
  await mesh("push");
  await new Promise((r) => setTimeout(r, 25_000));

  // Everything this test could want to know, gathered in one place.
  //
  // Three times now a diagnostic has not run because the thing before it threw: `must` on a
  // command that fails, and then a query that hangs long enough to take the harness's own timeout
  // with it. A 30-second test costs fifteen minutes to re-run, so the evidence has to be gathered
  // whether the failure is an assertion, an error, or a hang.
  const diagnose = async (machine: string) =>
    `--- ${machine}\n` +
    `dnsmasq: ${(await on(machine, `systemctl is-active dnsmasq.service`)).out.trim()}\n` +
    `${(await on(machine, `journalctl -u dnsmasq -n 12 --no-pager`)).out}\n` +
    `resolved: ${(await on(machine, `resolvectl status | head -30`)).out}\n` +
    `resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
    `what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
    `listening:\n${(await on(machine, `ss -lnup | grep :53 || echo none`)).out}\n` +
    `asked directly:\n${(await on(machine,
      `timeout 5 resolvectl query postgres.anchor.internal 2>&1 || echo "no answer"`)).out}`;

  // `on`, not `must`: `is-active` exits non-zero for a unit that failed, so `must` would throw
  // before the assertion below — taking every diagnostic with it. That happened, and the run said
  // only "failed".
  for (const machine of ["anchor", "laptop"]) {
    const state = await on(machine, `systemctl is-active dnsmasq.service`);
    if (state.out.trim() === "active") continue;
    assert.fail(`the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
      `its config:\n${(await on(machine, `cat /etc/dnsmasq.conf`)).out}\n` +
      `${await diagnose(machine)}`);
  }

  // Through the machine's own resolver, by the path an application actually takes: nsswitch, then
  // files, then DNS. `dig` would ask a server directly and prove less — the resolv.conf module is
  // half of what is being tested, and only this path goes through it.
  //
  // Bounded on the machine rather than by the harness: a query that hangs is a result, and letting
  // it run into the harness's own timeout turns it into an error with no evidence attached.
  const resolves = async (machine: string, name: string) => {
    const said = await on(machine,
      `timeout 5 getent hosts ${name} | head -1 | cut -d' ' -f1`, 20_000);
    return said.out.trim();
  };
  const addressOf = async (machine: string, node: string) =>
    (await must(machine, `getent hosts ${node}.internal | head -1 | cut -d' ' -f1`)).trim();

  const anchorAt = await addressOf("anchor", "anchor");
  const laptopAt = await addressOf("anchor", "laptop");

  // A name the mesh was never told about, under a machine it was — from both machines, because a
  // node must answer for every machine and not only for itself.
  for (const machine of ["anchor", "laptop"]) {
    let got = "";
    for (let i = 0; i < 15 && !got; i++) {
      got = await resolves(machine, "postgres.anchor.internal");
      if (!got) await new Promise((r) => setTimeout(r, 3000));
    }
    assert.equal(got, anchorAt,
      `${machine} does not resolve a service named under anchor: ${got || "(nothing)"}\n\n` +
      `${await diagnose(machine)}`);
  }

  // Any name at all, which is the whole point: the mesh was never told these exist.
  for (const [name, expected] of [
    ["postgres-2.anchor.internal", anchorAt],
    ["keycloak.anchor.internal", anchorAt],
    ["plex.laptop.internal", laptopAt],
    ["radarr.laptop.internal", laptopAt],
  ] as const) {
    assert.equal(await resolves("laptop", name), expected,
      `${name} did not resolve to the machine it is named under\n\n${await diagnose("laptop")}`);
  }

  // The machine's own name still resolves, and to the same place. Two accounts of where a machine
  // is, disagreeing, would be worse than either alone.
  assert.equal(await resolves("laptop", "anchor.internal"), anchorAt);

  // And what is not the mesh's is not answered by it. The resolver takes over the mesh's names
  // and nothing else, which is what lets a machine keep whatever DNS it already had.
  assert.equal(await resolves("anchor", "something.example.com"), "",
    "the resolver answered for a name that is not the mesh's");

  for (const machine of ["anchor", "laptop"]) {
    await mesh(`unassign ${machine} resolved-split-dns`);
    await mesh(`unassign ${machine} dnsmasq`);
  }
  await mesh("push");
});

// A real third-party workload, adopted the way the conversion will adopt one.
//
// **Everything before this used modules written to exercise the mesh.** This one is software
// nobody here wrote, taking its credentials the way such software does — from its environment —
// and needing two containers that reach each other by name. It is the first module that could not
// have been declared before today: it needs the `network` shape, and it needs a sealed value to
// reach a container's environment.
//
// Its database password is **accepted rather than generated**, which is the whole shape of an
// adoption: a service that already exists keeps the credential it already has, because minting a
// new one is how a running application stops being able to reach its own database.
test("a third-party workload is adopted, with the credential it already had", {
  skip, timeout: 900_000,
}, async () => {
  const password = "the-password-it-already-had";

  await must("anchor", `printf %s ${quote(JSON.stringify({
    module: "adopted-analytics",
    version: "1",
    capabilities: ["container-runtime"],
    "own-secrets": {
      database: "/var/lib/umami/database.env",
      app: "/var/lib/umami/app.env",
    },
    listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }],
    resources: [
      { id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" },
      // The two containers must reach each other by name, which is what this shape is for.
      { id: "net", type: "network", name: "umami" },
      {
        id: "db", type: "container", name: "umami-db",
        image: pinned("postgres"),
        network: "umami",
        env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" },
        "env-file": ["/var/lib/umami/database.env"],
      },
      {
        id: "app", type: "container", name: "umami",
        image: pinned("ghcr.io/umami-software/umami"),
        network: "umami",
        env: { DATABASE_TYPE: "postgresql" },
        "env-file": ["/var/lib/umami/app.env"],
        ports: ["1212:3000"],
      },
    ],
  }))} > /umami.json`);
  await must("anchor", `docker cp /umami.json mesh-controller:/umami.json`);
  await mesh("module add /umami.json");

  // **Accepted, not generated.** The value is what the database already answers to; the mesh
  // seals it and cannot read it again. Given whole, as the environment lines the containers read.
  await must("anchor",
    `printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
    `docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics database --from -`);
  await must("anchor",
    `printf %s ${quote(
      `DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
    `docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics app --from -`);

  await mesh("assign anchor adopted-analytics");
  await mesh("push anchor", 300_000);

  // Both containers, and the network they share.
  let up = false;
  for (let i = 0; i < 60 && !up; i++) {
    const running = await on("anchor", `docker ps --format '{{.Names}}'`);
    up = running.out.includes("umami-db") && running.out.includes("umami");
    if (!up) await new Promise((r) => setTimeout(r, 5000));
  }
  if (!up) {
    // Everything that could say why, gathered before asserting. "It did not start" is the one
    // thing already known; what is wanted is whether the mesh sent it, whether the host refused
    // it, and what the runtime said when it tried.
    const said = await mesh("status");
    const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`);
    const applied = await on("anchor",
      `${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`);
    const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` +
      `for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`);
    const tried = await on("anchor",
      `docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` +
      `docker logs umami-db 2>&1 | tail -15`);
    assert.fail(
      `the workload never started.\n\n` +
      `── what the mesh thinks:\n${said}\n` +
      `── containers:\n${containers.out}\n` +
      `── what the host owns:\n${applied.out}\n` +
      `── what the mesh wrote:\n${files.out}\n` +
      `── the database container:\n${tried.out}\n`);
  }

  // The environment file the mesh sealed is on the machine and readable only by root.
  const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`);
  assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root");

  // **The assertion that matters: the credential works.** Not that a file arrived — that the
  // database the mesh started answers to the password the mesh was given rather than one it made.
  let connected = { out: "", ok: false };
  for (let i = 0; i < 40 && !connected.ok; i++) {
    connected = await on("anchor",
      `docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`);
    if (!connected.ok) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(connected.ok, `the database never came up:\n${connected.out}`);

  const wrong = await on("anchor",
    `docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` +
    `psql -h umami-db -U umami -d umami -qAt -c 'select 1'`);
  assert.ok(!wrong.ok,
    "the database accepted a password nobody gave it, so this proves nothing about the one that was");

  // And the two containers reach each other by name over the module's own network.
  const reached = await must("anchor",
    `docker run --rm --network umami ${pinned("postgres")} ` +
    `sh -c 'getent hosts umami-db || echo unreachable'`);
  assert.doesNotMatch(reached, /unreachable/,
    "a container could not reach the other by name, so the module's network did nothing");
});

// The real modules, resolved together on one machine.
//
// **What this proves without pulling a gigabyte of images**: that five manifests written from the
// running system resolve as a graph — keycloak's requirement met by postgres's provision,
// capabilities checked, nothing claiming the same singular thing — and that the declaration the
// control plane composes is one the host accepts. `plan --json` exists for exactly this: it is
// the only way to know that what the control plane emits is what the host takes.
//
// Running them needs their images stocked and two provisioners built, which is a separate and
// larger job. This is the half that can be known now, and it is the half where a design fault
// would live.
// Three tests lived here that read the mesh's example modules, which moved to the catalogue.
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the
// catalogue's modules together and its gate is the composed declaration accepted and every core
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates
// on gitea running but does not yet ask it to answer on its port with the credential it was given,
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
// mesh-grant-end-to-end, against the catalogue's redis.