/** * Materialise the routers a declaration implies. * * A gateway is the one implicit machine in an otherwise explicit declaration — a scenario * says a segment sits behind one and never names the thing that serves it, because it has * nothing to say about it. * * A router is **scenery, not a node**, so it is a container rather than a virtual machine * (novox/hq ADR 0033). Nothing under test runs on it and no assertion is made about its * internals; it exists so packets behave the way they behave in the world. What it has to * reproduce is kernel behaviour, and a container has the same kernel. */ import type { Family, Scenario } from "../declaration/types.ts"; import { incus, succeeds } from "../incus/client.ts"; import { macFor, networkName } from "./names.ts"; import { waitUntilUsable } from "./ready.ts"; /** * The router image, built once and cached. * * A scenario is a closed address space, so a router has no route to a package repository — * installing nftables at raise time cannot work, and the first attempt failed exactly that * way. So the image is prepared once, with temporary connectivity, and every scenario * afterwards raises from it needing no network at all. * * That is the same property the mesh's own artifacts have: what ships is self-contained, * and a deploy touches no network. */ const ROUTER_IMAGE = "mesh-lab-router"; const ROUTER_BASE = "images:alpine/edge"; /** Wait until the container can actually resolve and fetch — not merely run a command. */ async function waitForNetwork(name: string, timeoutSeconds: number): Promise { const deadline = Date.now() + timeoutSeconds * 1000; let lastError = "no attempt made"; while (Date.now() < deadline) { try { await incus(["exec", name, "--", "apk", "update"], 30_000); return; } catch (err) { lastError = err instanceof Error ? err.message.split("\n")[0] ?? "" : String(err); } await new Promise((resolve) => setTimeout(resolve, 2000)); } throw new Error( `${name} had no working network after ${timeoutSeconds}s — the router image cannot be ` + `built without one. Last error: ${lastError}`, ); } /** * Build the router image if it is missing. One-time, and the only step in the whole lab that * needs the workstation to be online. */ export async function ensureRouterImage(log: (message: string) => void = () => {}): Promise { if (await succeeds(["image", "info", ROUTER_IMAGE], 20_000)) return; log(` building the router image (once) — installing nftables into ${ROUTER_BASE}`); const builder = "mlab-router-build"; await succeeds(["delete", "--force", builder], 60_000); // Default profile on purpose: this is the one container that needs to reach a repository. await incus(["launch", ROUTER_BASE, builder], 300_000); await waitUntilUsable(builder, 120, () => {}); // `exec` works before the container has an address. Usable means a command runs; it does // not mean the network is up, and the first attempt failed on DNS because those were // treated as the same thing. Wait for the thing actually needed. await waitForNetwork(builder, 60); // Not swallowed. A router without nftables is a router that silently does not route, and // an earlier attempt shipped exactly that because the failure was hidden behind `|| true`. await incus(["exec", builder, "--", "apk", "add", "--no-cache", "--update", "nftables"], 180_000); await incus(["exec", builder, "--", "sh", "-c", "command -v nft"], 20_000); // The stock image ships `auto eth0 / iface eth0 inet dhcp`, and its boot-time networking // service acts on it — flushing the static address the scenario just set, on eth0 only, // which is why the outside interface came up bare while the inside ones were fine. // // A scenario declares the underlay; a router that reconfigures itself from an image // default is the lab overriding the declaration. await incus([ "exec", builder, "--", "sh", "-c", "printf 'auto lo\\niface lo inet loopback\\n' > /etc/network/interfaces", ], 30_000); await incus(["stop", builder], 120_000); await incus(["publish", builder, "--alias", ROUTER_IMAGE], 300_000); await succeeds(["delete", "--force", builder], 60_000); log(` router image ready`); } /** * The address the transit router holds on a public segment: the last usable host address. * * Chosen rather than declared, like a gateway's inside address — a scenario has nothing to * say about the internet's own routers, only about the networks they connect. */ export function transitAddress(cidr: string): string | null { const slash = cidr.lastIndexOf("/"); if (slash === -1) return null; const base = cidr.slice(0, slash); const prefix = cidr.slice(slash); if (base.includes(":")) return `${base.replace(/::$/, "")}::fffe${prefix}`; const octets = base.split("."); octets[3] = "254"; return `${octets.join(".")}${prefix}`; } /** The name a router answers to in `list` and `exec` — scenery, but addressable. */ export function routerMachineName(plan: RouterPlan): string { return `gw-${plan.inside.join("-")}`; } export interface RouterPlan { /** Router name, one per distinct gateway. */ name: string; /** The segment(s) behind this router. Several share one when they share a gateway. */ inside: string[]; /** The segment this router reaches out through. */ outside: string; /** Addresses this router holds on the outside segment — what the world sees. */ outsideAddresses: string[]; nat: Family[]; forwardable: boolean; mappingTtl: string | undefined; } /** * Group segments by the gateway they declare. Identical gateway declarations mean ONE * router, not several — that is what a VLAN-capable router is, and two routers sharing an * external address would not work anyway. */ /** * Gateways that share an address are ONE gateway. * * Grouping on the exact address list instead split a household in two: `home` declaring a * v4 and a v6 address and `devices` declaring only the v4 produced two router containers, * both holding the same v4 address on the same segment. The lab raised it, and the shared * address resolved to whichever container answered ARP last — so a published port worked or * did not, run to run, with nothing reporting a fault. * * One public address is one box. Checked against the real thing this models: a bridged * modem, a single gateway holding the public address, everything behind it on one network. * Two routers on one address is not a topology, it is a collision. */ export function planRouters(scenario: Scenario, instanceId: string): RouterPlan[] { const plans: RouterPlan[] = []; for (const [segmentName, segment] of Object.entries(scenario.segments)) { const gateway = segment.gateway; if (!gateway) continue; const existing = plans.find( (plan) => plan.outside === gateway.to && plan.outsideAddresses.some((address) => gateway.address.includes(address)), ); if (existing) { existing.inside.push(segmentName); // The union, so a gateway declared with a v6 address on only one of the segments it // serves still carries it. The declarations must otherwise agree — validate refuses // the case where they do not, so there is nothing to reconcile here. for (const address of gateway.address) { if (!existing.outsideAddresses.includes(address)) existing.outsideAddresses.push(address); } continue; } plans.push({ name: `mlab-${instanceId}-gw${plans.length}`, inside: [segmentName], outside: gateway.to, outsideAddresses: [...gateway.address], nat: gateway.nat, forwardable: gateway.forwardable, mappingTtl: gateway.mappingTtl, }); } return plans; } /** "120s" / "2m" / "90" → seconds. */ export function ttlSeconds(text: string | undefined): number | undefined { if (!text) return undefined; const match = /^(\d+)\s*([smh]?)$/.exec(text.trim()); if (!match) return undefined; const value = Number(match[1]); return match[2] === "m" ? value * 60 : match[2] === "h" ? value * 3600 : value; } function withPrefix(scenario: Scenario, segment: string, address: string): string { const wantV6 = address.includes(":"); for (const range of scenario.segments[segment]?.cidr ?? []) { const slash = range.lastIndexOf("/"); if (slash === -1) continue; if (range.slice(0, slash).includes(":") === wantV6) return `${address}${range.slice(slash)}`; } return address; } /** The address a machine holds on a segment, for DNAT targets and as an inside gateway. */ function addressOn(scenario: Scenario, machine: string, segment: string, family: Family): string | null { const spec = scenario.machines[machine]; if (!spec || spec.at === "detached") return null; for (const attachment of spec.at) { if (attachment.segment !== segment) continue; for (const address of attachment.address) { if ((family === "v6") === address.includes(":")) return address; } } return null; } /** * The router's own address on an inside segment: the first host address of that range. * * Chosen rather than declared because a scenario has nothing to say about it — the * declaration describes what the world sees the network as, and the inside address is an * implementation detail of the machine serving it. */ function insideAddress(scenario: Scenario, segment: string, family: Family): string | null { for (const range of scenario.segments[segment]?.cidr ?? []) { const slash = range.lastIndexOf("/"); if (slash === -1) continue; const base = range.slice(0, slash); const isV6 = base.includes(":"); if (isV6 !== (family === "v6")) continue; if (isV6) return `${base.replace(/::$/, "::")}1${range.slice(slash)}`.replace("::1/", "::1/"); const octets = base.split("."); octets[3] = "1"; return `${octets.join(".")}${range.slice(slash)}`; } return null; } /** * Wire the public segments together. * * The internet is not a network — it is unrelated networks that route to each other, many * hops apart with no shared broadcast domain. So public segments are separate links joined * by a router, never bridged: bridging them would make ARP adjacency, non-decrementing TTL * and crossing multicast true in the lab and false in production, and the mesh has already * been bitten by multicast name resolution. * * One transit router, an interface on every public segment, forwarding and no translation. * It is the closest thing the lab has to "the internet", and it is deliberately dumb. */ export async function raiseTransit( scenario: Scenario, instanceId: string, log: (message: string) => void = () => {}, ): Promise { const publicSegments = Object.entries(scenario.segments) .filter(([, segment]) => segment.kind === "public") .map(([name]) => name); // One public network needs no transit: everything on it is already adjacent. if (publicSegments.length < 2) return null; const name = `mlab-${instanceId}-transit`; if (!(await succeeds(["config", "show", name], 15_000))) { await incus([ "init", ROUTER_IMAGE, name, "-c", `user.mesh-lab.instance=${instanceId}`, "-c", "user.mesh-lab.machine=transit", "-c", `user.mesh-lab.transit=${publicSegments.join(",")}`, ], 300_000); await succeeds(["config", "device", "remove", name, "eth0"], 15_000); for (const [index, segment] of publicSegments.entries()) { await incus([ "config", "device", "add", name, `eth${index}`, "nic", "nictype=bridged", `parent=${networkName(instanceId, segment)}`, `hwaddr=${macFor(instanceId, "transit", index)}`, ]); } } await succeeds(["start", name], 60_000); await waitUntilUsable(name, 120, () => {}); for (const [index, segment] of publicSegments.entries()) { const device = `eth${index}`; await sh(name, `ip link set ${device} up`); for (const cidr of scenario.segments[segment]?.cidr ?? []) { const address = transitAddress(cidr); if (address) await sh(name, `ip addr replace ${address} dev ${device}`); } const mtu = scenario.segments[segment]?.mtu; if (mtu) await sh(name, `ip link set ${device} mtu ${mtu}`); } await sh( name, "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null", ); log(` transit router across ${publicSegments.join(", ")}`); return name; } export async function raiseRouters( scenario: Scenario, instanceId: string, plans: RouterPlan[], log: (message: string) => void = () => {}, ): Promise { const created: string[] = []; if (plans.length > 0) await ensureRouterImage(log); for (const plan of plans) { if (!(await succeeds(["config", "show", plan.name], 15_000))) { await incus([ "init", ROUTER_IMAGE, plan.name, "-c", `user.mesh-lab.instance=${instanceId}`, // Tagged as a machine as well as a router: destroy finds an instance's resources // with one query, and a router that only carried `router=` was left behind — which // then held its networks open, so `destroy` reported removing zero segments. "-c", `user.mesh-lab.machine=${routerMachineName(plan)}`, "-c", `user.mesh-lab.router=${plan.inside.join(",")}`, // `outside` is structural — it is which link eth0 is on, true the moment the device // is added. The gateway's *behaviour* is not recorded here; see configureRouter. "-c", `user.mesh-lab.outside=${plan.outside}`, ], 300_000); await succeeds(["config", "device", "remove", plan.name, "eth0"], 15_000); // eth0 faces outward, then one interface per segment behind it. const links = [plan.outside, ...plan.inside]; for (const [index, segment] of links.entries()) { await incus([ "config", "device", "add", plan.name, `eth${index}`, "nic", "nictype=bridged", `parent=${networkName(instanceId, segment)}`, `hwaddr=${macFor(instanceId, `gw-${plan.name}`, index)}`, ]); } } await succeeds(["start", plan.name], 60_000); created.push(plan.name); log(` router ${plan.inside.join("+")} → ${plan.outside}`); } for (const name of created) { await waitUntilUsable(name, 120, () => {}); } for (const plan of plans) { await configureRouter(scenario, plan, log); } return created; } async function sh(name: string, script: string, timeoutMs = 60_000): Promise { await incus(["exec", name, "--", "sh", "-c", script], timeoutMs); } async function configureRouter( scenario: Scenario, plan: RouterPlan, log: (message: string) => void, ): Promise { // Addresses: eth0 outside, then one per inside segment. const links: { device: string; segment: string; addresses: string[] }[] = [ { device: "eth0", segment: plan.outside, addresses: plan.outsideAddresses.map((a) => withPrefix(scenario, plan.outside, a)), }, ]; for (const [index, segment] of plan.inside.entries()) { const addresses: string[] = []; for (const family of ["v4", "v6"] as Family[]) { const address = insideAddress(scenario, segment, family); if (address) addresses.push(address); } links.push({ device: `eth${index + 1}`, segment, addresses }); } for (const link of links) { // Up first: an address on a down interface is accepted and then not used. await sh(plan.name, `ip link set ${link.device} up`); for (const address of link.addresses) { // `replace` rather than `add`, so re-running is safe and a real failure still fails. await sh(plan.name, `ip addr replace ${address} dev ${link.device}`); } const mtu = scenario.segments[link.segment]?.mtu; if (mtu) await sh(plan.name, `ip link set ${link.device} mtu ${mtu}`); } await sh( plan.name, "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null", ); // A gateway reaches other public networks the way anything does: through transit. Without // this it can only reach its own outside segment, and every scenario with more than one // public network becomes a set of islands. for (const cidr of scenario.segments[plan.outside]?.cidr ?? []) { const via = transitAddress(cidr); if (!via) continue; const gateway = via.slice(0, via.lastIndexOf("/")); const family = gateway.includes(":") ? "-6" : "-4"; await sh(plan.name, `ip ${family} route replace default via ${gateway} dev eth0 2>/dev/null || true`); } const ttl = ttlSeconds(plan.mappingTtl); if (ttl !== undefined) { // What makes keepalive behaviour testable rather than hoped for: a connection held // through NAT without refreshing dies when the mapping does. // // Read back rather than assumed. These sysctls are not present on every kernel, and a // scenario that declared an expiring mapping and silently got a permanent one would be // the fault this lab exists to catch. await sh( plan.name, `sysctl -w net.netfilter.nf_conntrack_udp_timeout=${ttl} >/dev/null 2>&1; ` + `sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=${ttl} >/dev/null 2>&1; true`, ); const readback = await incus( ["exec", plan.name, "--", "sh", "-c", "cat /proc/sys/net/netfilter/nf_conntrack_udp_timeout 2>/dev/null || echo missing"], 20_000, ); if (readback.stdout.trim() !== String(ttl)) { throw new Error( `${plan.name}: mapping_ttl of ${plan.mappingTtl} was declared but conntrack reports ` + `'${readback.stdout.trim()}'. The scenario would silently have permanent mappings.`, ); } } await applyRules(scenario, plan); // Recorded last, and only here. Everything above either read itself back or threw, so a // gateway carrying these tags is one that demonstrably does these things. Written at // `init` they would have been a restatement of the request — and a raise that failed // half way leaves its wreckage standing on purpose, so a picture of that wreckage would // have badged translation the router was never configured to do. const recorded = [ `user.mesh-lab.nat=${plan.nat.join(",")}`, `user.mesh-lab.forwardable=${plan.forwardable}`, ...(ttl === undefined ? [] : [`user.mesh-lab.mapping-ttl=${ttl}`]), ]; for (const entry of recorded) { const at = entry.indexOf("="); await succeeds(["config", "set", plan.name, entry.slice(0, at), entry.slice(at + 1)], 20_000); } log(` ${plan.name}: nat=${plan.nat.join(",") || "none"} forwardable=${plan.forwardable}${ttl ? ` ttl=${ttl}s` : ""}`); } /** One ruleset per router, written whole — partial rule edits drift, a whole file does not. */ async function applyRules(scenario: Scenario, plan: RouterPlan): Promise { const parts: string[] = ["flush ruleset"]; for (const family of plan.nat) { const table = family === "v4" ? "ip" : "ip6"; parts.push( `table ${table} nat {`, ` chain postrouting { type nat hook postrouting priority srcnat; policy accept;`, ` oifname "eth0" masquerade`, ` }`, ` chain prerouting { type nat hook prerouting priority dstnat; policy accept;`, ); if (plan.forwardable) { for (const [machine, spec] of Object.entries(scenario.machines)) { for (const publication of spec.published ?? []) { if (!plan.inside.includes(publication.on)) continue; const target = addressOn(scenario, machine, publication.on, family); if (!target) continue; const destination = family === "v6" ? `[${target}]` : target; parts.push( ` iifname "eth0" tcp dport ${publication.port} dnat to ${destination}:${publication.port}`, ); } } } parts.push(` }`, `}`); } // Filtering: unsolicited inbound, and policy between segments the router serves. const filterLines: string[] = []; if (!plan.forwardable) { // A gateway you do not control: outbound works, nothing initiates inward. That is the // constraint being reproduced, not an implementation limit. filterLines.push(` iifname "eth0" ct state new drop`); } for (const rule of scenario.policy ?? []) { if (rule.allow) continue; const fromIndex = plan.inside.indexOf(rule.from); const toIndex = plan.inside.indexOf(rule.to); if (fromIndex === -1 || toIndex === -1) continue; filterLines.push(` iifname "eth${fromIndex + 1}" oifname "eth${toIndex + 1}" drop`); } if (filterLines.length > 0) { parts.push( `table inet filter {`, ` chain forward { type filter hook forward priority filter; policy accept;`, ` ct state established,related accept`, ...filterLines, ` }`, `}`, ); } const ruleset = parts.join("\n"); await sh( plan.name, `cat > /tmp/mlab.nft <<'MLABNFT'\n${ruleset}\nMLABNFT\nnft -f /tmp/mlab.nft`, 60_000, ); }