diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index 87d2c4b..cb62fdf 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -16,9 +16,16 @@ machines: anchor: at: { segment: hosting, address: [192.0.2.10] } inbound: allow + # The whole substrate, the registry, the builder, an adopted workload and the modules under + # test all land here — eleven containers before the forge arrives. At the 1GiB default this + # machine thrashes, and it presents as "the mesh hangs": every exec slows from 15s to 105s + # and the forge test fails on a status poll that is merely queued behind page-outs. + memory: 4GiB + cpus: 4 laptop: at: { segment: hosting, address: [192.0.2.20] } inbound: allow + memory: 2GiB images: - postgres:17-alpine diff --git a/src/declaration/parse.ts b/src/declaration/parse.ts index d38fc24..0911ae0 100644 --- a/src/declaration/parse.ts +++ b/src/declaration/parse.ts @@ -37,6 +37,9 @@ function normaliseMachine(raw: unknown): Machine { } const inbound = machine["inbound"]; if (inbound === "allow" || inbound === "deny") result.inbound = inbound; + if (machine["egress"] === true) result.egress = true; + if (machine["memory"] !== undefined) result.memory = String(machine["memory"]); + if (machine["cpus"] !== undefined) result.cpus = Number(machine["cpus"]); return result; } diff --git a/src/declaration/types.ts b/src/declaration/types.ts index b4e5e4e..8b70f04 100644 --- a/src/declaration/types.ts +++ b/src/declaration/types.ts @@ -85,6 +85,31 @@ export interface Machine { * v6-addressed machine. Without this, v6 addressing would imply reachability. */ inbound?: "allow" | "deny"; + /** + * Whether this machine can reach the world outside the scenario. + * + * **Off unless asked for.** A scenario is a closed address space, and a machine that could + * reach anything would make every test's result depend on what else was reachable that day. + * It is declared for the same reason an address is: so what a run proves is what the scenario + * says, and not what the workstation happened to have. + * + * What it is for is the one thing a mesh genuinely cannot do without an outside: a first node + * fetching the images it starts from, before there is any mesh to serve them + * (novox/hq 04-ISSUES/029). + */ + egress?: boolean; + /** + * How big the machine is. Absent means the lab's default, which suits a machine running a host + * and a handful of containers. + * + * Declared, because it is a fact about the machine the scenario describes — the node that runs + * the whole substrate is bigger than the laptop that joins it, and a test that starves its + * anchor at the default answers questions about memory pressure, not about the mesh. The forge + * test failed three times as "status hangs" before anyone counted the containers in 1GiB + * (novox/hq 04-ISSUES/024 is the same lesson about a different resource). + */ + memory?: string; + cpus?: number; } /** Reachability between segments, as a segmented router enforces it. Asymmetric by design. */ diff --git a/src/declaration/validate.ts b/src/declaration/validate.ts index 20fbdd1..8f772e0 100644 --- a/src/declaration/validate.ts +++ b/src/declaration/validate.ts @@ -204,6 +204,12 @@ export function validate(scenario: Scenario): void { if (machine.published?.length) { problems.push(`machine '${name}' is detached but declares published ports`); } + // The same fault as publishing from nowhere: raising it would drop the key on the floor, + // and a scenario key the runtime silently ignores is the thing this lab exists to catch. + if (machine.egress) { + problems.push(`machine '${name}' is detached but declares egress — a machine on no ` + + `segment reaches nothing, the world included`); + } continue; } diff --git a/src/lifecycle/address.ts b/src/lifecycle/address.ts index 917caff..8af4143 100644 --- a/src/lifecycle/address.ts +++ b/src/lifecycle/address.ts @@ -23,6 +23,15 @@ export interface Wire { mac: string; addresses: string[]; mtu: number | undefined; + /** + * Ask the host for an address rather than declaring one. + * + * **Only the uplink**, and it is the one address a scenario has no business choosing: it is on + * the machine's side of the host's own network, and what is routable there is the host's fact, + * not the declaration's. Every segment a scenario describes is still static, for the reason + * below. + */ + dhcp?: boolean; } /** @@ -31,6 +40,24 @@ export interface Wire { * the declaration is supposed to own. */ function networkUnit(wire: Wire): string { + if (wire.dhcp) { + return [ + "[Match]", + `MACAddress=${wire.mac}`, + "", + "[Network]", + "DHCP=ipv4", + "IPv6AcceptRA=no", + "", + "[DHCPv4]", + // **Worse than any route the scenario states.** A machine behind a declared gateway must + // keep using it: the uplink is a way out of the scenario, not a better way around inside + // it. On-link segments win regardless, being connected routes; this only settles which + // default route is preferred when a scenario declares one of its own. + "RouteMetric=4096", + "UseDNS=yes", + ].join("\n") + "\n"; + } const lines = [ "[Match]", `MACAddress=${wire.mac}`, @@ -98,6 +125,16 @@ export async function applyAddresses( }); } + if (spec.egress) { + wires.push({ + device: `eth${spec.at.length}`, + mac: macFor(instanceId, machine, spec.at.length), + addresses: [], + mtu: undefined, + dhcp: true, + }); + } + for (const [index, wire] of wires.entries()) { const unit = networkUnit(wire); await incus( @@ -109,7 +146,7 @@ export async function applyAddresses( await incus(["exec", name, "--", "systemctl", "enable", "--now", "systemd-networkd"], 60_000); await incus(["exec", name, "--", "systemctl", "restart", "systemd-networkd"], 60_000); - log(` addressed ${machine} (${wires.map((w) => w.addresses.join(",")).join(" | ")})`); + log(` addressed ${machine} (${wires.map((w) => w.dhcp ? "uplink:dhcp" : w.addresses.join(",")).join(" | ")})`); } } diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 3cb4374..9ebc0ee 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -131,12 +131,42 @@ async function createNetwork( return name; } +/** + * The one network the lab supplies rather than the declaration. + * + * Every segment a scenario describes is an isolated bridge with no addresses, no DHCP and no NAT, + * because the declaration owns addressing. This is the opposite of that on purpose: it is not part + * of the scenario, it carries no scenario traffic, and what is routable on it is the host's fact. + * + * It exists so a machine can fetch what it starts from. A first node pulls three images before + * there is any mesh, and the module that gives a mesh its own store pulls one more + * (novox/hq 04-ISSUES/029) — none of which anything inside a scenario can serve. + * + * Tagged like everything else, so tearing the scenario down takes it too. + */ +async function createUplink(instanceId: string): Promise { + const name = networkName(instanceId, "uplink"); + if (await succeeds(["network", "show", name], 15_000)) return name; + await incus([ + "network", "create", name, + "ipv4.address=auto", + "ipv4.nat=true", + "ipv6.address=none", + `user.mesh-lab.instance=${instanceId}`, + "user.mesh-lab.segment=uplink", + ]); + return name; +} + async function createMachine( instanceId: string, machine: string, attachments: { segment: string }[], image: string, pool: string, + egress = false, + memory = "1GiB", + cpus = 2, ): Promise { const name = machineName(instanceId, machine); if (await succeeds(["config", "show", name], 15_000)) return name; @@ -148,8 +178,8 @@ async function createMachine( // Arch images refuse to boot under secureboot with the shipped keys. Discovered by // the first launch failing with exactly that message. "-c", "security.secureboot=false", - "-c", "limits.memory=1GiB", - "-c", "limits.cpu=2", + "-c", `limits.memory=${memory}`, + "-c", `limits.cpu=${cpus}`, "-c", `user.mesh-lab.instance=${instanceId}`, "-c", `user.mesh-lab.machine=${machine}`, ]; @@ -168,6 +198,17 @@ async function createMachine( `hwaddr=${macFor(instanceId, machine, index)}`, ]); } + // After every declared attachment, so eth0..ethN keep meaning what the scenario said and the + // uplink is whatever comes next. A machine that never asked for one has no such interface at + // all, which is the difference between a closed scenario and an open one. + if (egress) { + await incus([ + "config", "device", "add", name, `eth${attachments.length}`, "nic", + "nictype=bridged", + `parent=${await createUplink(instanceId)}`, + `hwaddr=${macFor(instanceId, machine, attachments.length)}`, + ]); + } return name; } @@ -242,7 +283,9 @@ export async function raise( const byMachine = new Map(); for (const [machine, spec] of Object.entries(scenario.machines)) { const attachments = spec.at === "detached" ? [] : spec.at; - const name = await createMachine(instanceId, machine, attachments, image, pool); + const name = await createMachine( + instanceId, machine, attachments, image, pool, + spec.at !== "detached" && spec.egress === true, spec.memory, spec.cpus); created.push(name); byMachine.set(machine, name); log(` machine ${machine}${spec.at === "detached" ? " (detached)" : ""}`); diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index a593edb..8aff77c 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -128,8 +128,16 @@ async function settled(node: string, withinMs = 240_000): Promise { // second is this machine's fault. The control plane is a container on the node being polled: // while it applies a declaration, an exec into it can lose its fifo to containerd, and a poll // loop that treats that as a verdict reports the mesh broken because the question missed. - const { out: said, ok } = await on("anchor", - `docker exec mesh-control /mesh-control status --json`); + // And a poll that *threw* — an exec timeout, a lost fifo — is also "could not ask", not a + // verdict. The distinction failed once as an IncusError surfacing at minute four of a wait + // whose machine was merely slow. + let said = "", ok = false; + try { + ({ out: said, ok } = await on("anchor", + `docker exec mesh-control /mesh-control status --json`)); + } catch (err) { + said = (err as Error).message; + } if (!ok) { last = said; await new Promise((r) => setTimeout(r, 5000)); diff --git a/test/supported.test.ts b/test/supported.test.ts index 410e89b..bda3058 100644 --- a/test/supported.test.ts +++ b/test/supported.test.ts @@ -84,3 +84,13 @@ segments: { net: { kind: public, cidr: [192.0.2.0/24] } } machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: allow } }`); assert.doesNotThrow(() => assertSupported(scenario)); }); + +test("egress is parsed, and off unless asked for", () => { + const scenario = parseScenario(`scenario: x +segments: { net: { kind: public, cidr: [192.0.2.0/24] } } +machines: + a: { at: { segment: net, address: [192.0.2.1] }, egress: true } + b: { at: { segment: net, address: [192.0.2.2] } }`); + assert.equal(scenario.machines["a"]?.egress, true); + assert.equal(scenario.machines["b"]?.egress, undefined); +}); diff --git a/test/validate.test.ts b/test/validate.test.ts index 1256c1a..b928d07 100644 --- a/test/validate.test.ts +++ b/test/validate.test.ts @@ -243,3 +243,10 @@ machines: /one gateway.*disagree.*forwardable/s, ); }); + +test("a detached machine cannot declare egress", () => { + refuses(`scenario: x +segments: { net: { kind: public, cidr: [192.0.2.0/24] } } +machines: { a: { at: detached, egress: true } }`, + /detached but declares egress/); +});