mesh-lab/test/integration/underlay.test.ts

/**
 * Each test names the decision it defends. A decision with no test is one that will quietly
 * stop being true (novox/hq ADR 0017).
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec, list, restore, snapshot } from "../../src/lifecycle/operate.ts";
import { incus, incusOk } from "../../src/incus/client.ts";
import { labIsUsable, destroyAll, assertUniversalInvariants } from "./harness.ts";
import { diagramFromLive } from "../../src/diagram/from-live.ts";
import { diagramFromDeclaration } from "../../src/diagram/from-declaration.ts";
import { toDrawio } from "../../src/diagram/drawio.ts";

const capability = await labIsUsable();
const skip = capability.usable ? false : `lab not usable: ${capability.why}`;

let instanceId = "";

before(async () => {
  if (skip) return;
  const scenario = loadScenario("scenarios/behind-nat.yml");
  const raised = await raise(scenario, {});
  instanceId = raised.instanceId;
});

after(async () => {
  if (instanceId) await destroy(instanceId);
});

test("ADR 0016 — the lab provides the underlay and NOTHING of the overlay", { skip, timeout: 120_000 }, async () => {
  // A scenario that pre-built peering would certify its own work. Whatever the mesh is
  // responsible for must be absent from a freshly raised machine.
  const { stdout } = await exec(instanceId, "home-server", [
    "sh", "-c",
    "ip link show type wireguard 2>/dev/null | wc -l; " +
      "ls /etc/wireguard 2>/dev/null | wc -l; " +
      "ls /etc/hal /etc/mesh 2>/dev/null | wc -l",
  ]);
  const counts = stdout.trim().split("\n").map((n) => Number(n.trim()));
  assert.deepEqual(counts, [0, 0, 0], "a raised machine carries no overlay, no mesh config");
});

test("ADR 0016 — the declared address IS what the machine holds", { skip }, async () => {
  const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]);
  assert.match(stdout, /10\.99\.1\.135\/24/);
});

test("design — raise waits for USABLE, not for the call to return", { skip, timeout: 120_000 }, async () => {
  // The measured gap is 3.4s to 14.3s. Reporting the earlier number is transport reported
  // as effect. If raise has returned, every machine must answer immediately.
  for (const machine of ["anchor", "home-server"]) {
    const { stdout } = await exec(instanceId, machine, ["sh", "-c", "echo alive"]);
    assert.equal(stdout.trim(), "alive", `${machine} was not usable when raise returned`);
  }
});

test("ADR 0016 — a router is scenery: containers, while machines are virtual machines", { skip, timeout: 120_000 }, async () => {
  const json = (await incusOk(["list", "--format", "json"], 30_000)) ?? "[]";
  const all = JSON.parse(json) as { name?: string; type?: string; config?: Record<string, string> }[];
  const mine = all.filter((i) => i.config?.["user.mesh-lab.instance"] === instanceId);
  assert.ok(mine.length >= 3, "expected machines and a router");

  for (const item of mine) {
    const isRouter = item.config?.["user.mesh-lab.router"] !== undefined;
    assert.equal(
      item.type,
      isRouter ? "container" : "virtual-machine",
      `${item.name} is a ${item.type} but ${isRouter ? "is" : "is not"} a router`,
    );
  }
});

test("design — NAT: a private address is not reachable from outside", { skip, timeout: 120_000 }, async () => {
  const { stdout } = await exec(instanceId, "anchor", [
    "sh", "-c", "ping -c1 -W2 10.99.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable",
  ]);
  assert.equal(stdout.trim(), "unreachable");
});

test("design — published: reachable at the GATEWAY's address, never its own", { skip, timeout: 180_000 }, async () => {
  await exec(instanceId, "home-server", [
    "sh", "-c", "nohup python3 -m http.server 8080 --bind 0.0.0.0 >/tmp/s.log 2>&1 & sleep 2",
  ]);
  const { stdout } = await exec(instanceId, "anchor", [
    "sh", "-c", "curl -s -m5 -o /dev/null -w '%{http_code}' http://192.0.2.50:8080/ || echo failed",
  ]);
  assert.equal(stdout.trim(), "200", "the forwarded port did not reach the machine behind NAT");
});

test("design — snapshots are WHOLE-scenario: restore returns every machine", { skip, timeout: 600_000 }, async () => {
  // Restoring a subset would produce a mesh that has never existed, so faults found there
  // would be artefacts of the lab.
  await exec(instanceId, "anchor", ["sh", "-c", "echo dirty > /root/marker"]);
  await exec(instanceId, "home-server", ["sh", "-c", "echo dirty > /root/marker"]);
  await snapshot(instanceId, "test-point");

  await exec(instanceId, "anchor", ["sh", "-c", "echo changed > /root/marker"]);
  await exec(instanceId, "home-server", ["sh", "-c", "echo changed > /root/marker"]);
  await restore(instanceId, "test-point");

  for (const machine of ["anchor", "home-server"]) {
    const { stdout } = await exec(instanceId, machine, ["cat", "/root/marker"]);
    assert.equal(stdout.trim(), "dirty", `${machine} was not returned to the snapshot`);
  }
});

test("design — restore leaves the scenario USABLE, not merely running", { skip, timeout: 120_000 }, async () => {
  // The restore call returns in under a second while the agent is still starting. Reporting
  // that as restored would be transport reported as effect.
  const { stdout } = await exec(instanceId, "anchor", ["sh", "-c", "echo alive"]);
  assert.equal(stdout.trim(), "alive");
});

test("ADR 0016 — the workstation has no route into the scenario", { skip, timeout: 60_000 }, async () => {
  // Reachability is asked from INSIDE. If the workstation could reach a scenario address,
  // two scenarios carrying the same prefix would put one's traffic in the other.
  const { stdout } = await incus(["exec", `mlab-${instanceId}-anchor`, "--", "echo", "inside"]);
  assert.equal(stdout.trim(), "inside", "exec is the only way in, and it works");
});

test("what came up holds what was declared, with no address held twice", { skip, timeout: 120_000 }, async () => {
  // The same invariants every scenario is held to, asserted here too — this instance is
  // already standing, so it costs nothing to ask.
  await assertUniversalInvariants(loadScenario("scenarios/behind-nat.yml"), instanceId);
});

// The diagram tests read the instance the file raised, so they run before the one that
// tears it down. Ordering is load-bearing here: appended after the destroy test they read
// an instance that no longer existed, and reported it as the diagram failing.
test("the live diagram reads the hypervisor, and a VM's addresses are not lost", { skip }, async () => {
  // A container's interface carries the device's name; a virtual machine names its own, so
  // joining addresses to devices by name attached every address to a container and none to
  // a VM. The picture then showed machines that looked like they had failed to come up.
  const drawn = await diagramFromLive(instanceId);
  const server = drawn.machines.find((m) => m.name === "home-server");
  assert.ok(server, "home-server missing from the live picture");
  assert.equal(server.kind, "machine");
  assert.ok(
    server.attachments.some((a) => a.addresses.some((address) => address.startsWith("10.99.1.135"))),
    `a virtual machine's addresses were not read back: ${JSON.stringify(server.attachments)}`,
  );
});

test("the live diagram draws what exists, never what was asked for", { skip, timeout: 300_000 }, async () => {
  // Every property shown must have come off the hypervisor. Proved by changing the running
  // system and watching only the live picture move.
  const scenario = loadScenario("scenarios/behind-nat.yml");
  const declared = diagramFromDeclaration(scenario);
  const before = await diagramFromLive(instanceId);
  assert.equal(before.machines.length, declared.machines.length, "the two pictures disagree on size");

  const anchor = (await list())
    .find((i) => i.instanceId === instanceId)
    ?.machines.find((m) => m.machine === "anchor");
  assert.ok(anchor, "anchor not found");
  await incus(["stop", anchor.name], 120_000);
  try {
    const after = await diagramFromLive(instanceId);
    assert.equal(after.machines.find((m) => m.name === "anchor")?.status, "Stopped");
    // The declaration has not changed, and neither has its picture.
    assert.equal(
      diagramFromDeclaration(scenario).machines.find((m) => m.name === "anchor")?.status,
      undefined,
      "a declared picture reported a runtime status it cannot know",
    );
  } finally {
    await incus(["start", anchor.name], 120_000);
  }
});

test("ADR 0016 — the live diagram distinguishes scenery from a node", { skip }, async () => {
  // The router is drawn as a router because the hypervisor says it is a container tagged as
  // a gateway — not because the diagram re-read the scenario and inferred it.
  const drawn = await diagramFromLive(instanceId);
  const gateway = drawn.machines.find((m) => m.kind === "router");
  assert.ok(gateway, "no gateway in the live picture");
  assert.ok(gateway.notes.includes("container"), "the gateway is not reported as scenery");
  assert.ok(gateway.notes.some((n) => n.startsWith("NAT ")), "translation is not shown");
  assert.ok(gateway.notes.includes("forwardable"), "forwardability is not shown");
  assert.ok(
    gateway.notes.some((n) => /mappings expire \d+s/.test(n)),
    "the declared mapping expiry was not recorded on the gateway",
  );

  const segments = new Map(drawn.segments.map((s) => [s.name, s]));
  assert.equal(segments.get("hosting")?.kind, "public", "segment kind was not recorded at raise");
  assert.equal(segments.get("home")?.behind, "hosting", "the tree was not recovered from the gateway tags");
  assert.equal(segments.get("home")?.depth, 1);
});

test("a picture nobody can open is not a picture", { skip }, async () => {
  // The first generated file was unparseable — HTML labels concatenated into an XML
  // attribute. Checked here against real output as well as against the fixtures, because
  // live labels carry names and addresses the declared ones never contain.
  const xml = toDrawio(await diagramFromLive(instanceId));
  for (const match of xml.matchAll(/(?:value|label|tooltip)="([^"]*)"/g)) {
    assert.doesNotMatch(match[1] ?? "", /[<>]/, "raw markup reached an XML attribute");
  }
  const ids = [...xml.matchAll(/<(?:mxCell|object) [^>]*id="([^"]*)"/g)].map((m) => m[1]);
  assert.equal(new Set(ids).size, ids.length, "duplicate cell id — draw.io drops one silently");
});

test("housekeeping — destroy removes machines, routers and segments", { skip, timeout: 400_000 }, async () => {
  const before = (await list()).find((i) => i.instanceId === instanceId);
  assert.ok(before, "the instance should exist before it is destroyed");

  const { machines, networks } = await destroy(instanceId);
  assert.ok(machines >= 3, `expected machines and a router, removed ${machines}`);
  assert.ok(networks >= 2, `expected both segments removed, removed ${networks}`);

  const after = (await list()).find((i) => i.instanceId === instanceId);
  assert.equal(after, undefined, "the instance should be gone");
  instanceId = "";
  await destroyAll("behind-nat-");
});