Files
mesh-lab/src/lifecycle/router.ts
T
jschoubben a6b7d67e19 Gateways sharing an address are one gateway
Found by asking what gw-devices and gw-home actually were, in a picture that
finally made them easy to see side by side.

planRouters grouped on the exact address list, so `home` declaring a v4 and a v6
address and `devices` declaring only the v4 became two router containers — both
holding 198.51.100.7 on the same segment. The lab raised it without complaint.

Not theoretical. On the raised instance the transit router resolved that one
address to two different MACs across a cache flush:

    198.51.100.7 -> 02:c9:16:70:23:29   (gw0, which HAS the :443 dnat)
    198.51.100.7 -> 02:bd:75:0b:b0:75   (gw1, which has none)

So home-server's published port worked or did not depending on which container
answered ARP last — intermittent, and it would have presented as a flaky test
rather than as a broken scenario.

One public address is one box. Checked against the thing this models rather than
argued from the model: a bridged modem, a single gateway holding the public
address, one network behind it, and every port forward landing on one host at
that address. Two routers on one address is not a topology, it is a collision.

Gateways to the same segment sharing any address are now one router and their
address lists union, so a v6 address declared on only one of the segments it
serves is still carried. Where such declarations disagree on nat, forwardable or
mapping_ttl, validate refuses — one box cannot behave two ways.

the-ordinary-shape now raises 7 machines instead of 8, and gw0 holds the public
address on eth0 while serving home on eth1 and devices on eth2.
2026-08-24 23:43:23 +02:00

518 lines
21 KiB
TypeScript

/**
* Materialise the routers a declaration implies.
*
* A gateway is the one implicit machine in an otherwise explicit declaration — a scenario
* says a segment sits behind one and never names the thing that serves it, because it has
* nothing to say about it.
*
* A router is **scenery, not a node**, so it is a container rather than a virtual machine
* (novox/hq ADR 0033). Nothing under test runs on it and no assertion is made about its
* internals; it exists so packets behave the way they behave in the world. What it has to
* reproduce is kernel behaviour, and a container has the same kernel.
*/
import type { Family, Scenario } from "../declaration/types.ts";
import { incus, succeeds } from "../incus/client.ts";
import { macFor, networkName } from "./names.ts";
import { waitUntilUsable } from "./ready.ts";
/**
* The router image, built once and cached.
*
* A scenario is a closed address space, so a router has no route to a package repository —
* installing nftables at raise time cannot work, and the first attempt failed exactly that
* way. So the image is prepared once, with temporary connectivity, and every scenario
* afterwards raises from it needing no network at all.
*
* That is the same property the mesh's own artifacts have: what ships is self-contained,
* and a deploy touches no network.
*/
const ROUTER_IMAGE = "mesh-lab-router";
const ROUTER_BASE = "images:alpine/edge";
/** Wait until the container can actually resolve and fetch — not merely run a command. */
async function waitForNetwork(name: string, timeoutSeconds: number): Promise<void> {
const deadline = Date.now() + timeoutSeconds * 1000;
let lastError = "no attempt made";
while (Date.now() < deadline) {
try {
await incus(["exec", name, "--", "apk", "update"], 30_000);
return;
} catch (err) {
lastError = err instanceof Error ? err.message.split("\n")[0] ?? "" : String(err);
}
await new Promise((resolve) => setTimeout(resolve, 2000));
}
throw new Error(
`${name} had no working network after ${timeoutSeconds}s — the router image cannot be ` +
`built without one. Last error: ${lastError}`,
);
}
/**
* Build the router image if it is missing. One-time, and the only step in the whole lab that
* needs the workstation to be online.
*/
export async function ensureRouterImage(log: (message: string) => void = () => {}): Promise<void> {
if (await succeeds(["image", "info", ROUTER_IMAGE], 20_000)) return;
log(` building the router image (once) — installing nftables into ${ROUTER_BASE}`);
const builder = "mlab-router-build";
await succeeds(["delete", "--force", builder], 60_000);
// Default profile on purpose: this is the one container that needs to reach a repository.
await incus(["launch", ROUTER_BASE, builder], 300_000);
await waitUntilUsable(builder, 120, () => {});
// `exec` works before the container has an address. Usable means a command runs; it does
// not mean the network is up, and the first attempt failed on DNS because those were
// treated as the same thing. Wait for the thing actually needed.
await waitForNetwork(builder, 60);
// Not swallowed. A router without nftables is a router that silently does not route, and
// an earlier attempt shipped exactly that because the failure was hidden behind `|| true`.
await incus(["exec", builder, "--", "apk", "add", "--no-cache", "--update", "nftables"], 180_000);
await incus(["exec", builder, "--", "sh", "-c", "command -v nft"], 20_000);
// The stock image ships `auto eth0 / iface eth0 inet dhcp`, and its boot-time networking
// service acts on it — flushing the static address the scenario just set, on eth0 only,
// which is why the outside interface came up bare while the inside ones were fine.
//
// A scenario declares the underlay; a router that reconfigures itself from an image
// default is the lab overriding the declaration.
await incus([
"exec", builder, "--", "sh", "-c",
"printf 'auto lo\\niface lo inet loopback\\n' > /etc/network/interfaces",
], 30_000);
await incus(["stop", builder], 120_000);
await incus(["publish", builder, "--alias", ROUTER_IMAGE], 300_000);
await succeeds(["delete", "--force", builder], 60_000);
log(` router image ready`);
}
/**
* The address the transit router holds on a public segment: the last usable host address.
*
* Chosen rather than declared, like a gateway's inside address — a scenario has nothing to
* say about the internet's own routers, only about the networks they connect.
*/
export function transitAddress(cidr: string): string | null {
const slash = cidr.lastIndexOf("/");
if (slash === -1) return null;
const base = cidr.slice(0, slash);
const prefix = cidr.slice(slash);
if (base.includes(":")) return `${base.replace(/::$/, "")}::fffe${prefix}`;
const octets = base.split(".");
octets[3] = "254";
return `${octets.join(".")}${prefix}`;
}
/** The name a router answers to in `list` and `exec` — scenery, but addressable. */
export function routerMachineName(plan: RouterPlan): string {
return `gw-${plan.inside.join("-")}`;
}
export interface RouterPlan {
/** Router name, one per distinct gateway. */
name: string;
/** The segment(s) behind this router. Several share one when they share a gateway. */
inside: string[];
/** The segment this router reaches out through. */
outside: string;
/** Addresses this router holds on the outside segment — what the world sees. */
outsideAddresses: string[];
nat: Family[];
forwardable: boolean;
mappingTtl: string | undefined;
}
/**
* Group segments by the gateway they declare. Identical gateway declarations mean ONE
* router, not several — that is what a VLAN-capable router is, and two routers sharing an
* external address would not work anyway.
*/
/**
* Gateways that share an address are ONE gateway.
*
* Grouping on the exact address list instead split a household in two: `home` declaring a
* v4 and a v6 address and `devices` declaring only the v4 produced two router containers,
* both holding the same v4 address on the same segment. The lab raised it, and the shared
* address resolved to whichever container answered ARP last — so a published port worked or
* did not, run to run, with nothing reporting a fault.
*
* One public address is one box. Checked against the real thing this models: a bridged
* modem, a single gateway holding the public address, everything behind it on one network.
* Two routers on one address is not a topology, it is a collision.
*/
export function planRouters(scenario: Scenario, instanceId: string): RouterPlan[] {
const plans: RouterPlan[] = [];
for (const [segmentName, segment] of Object.entries(scenario.segments)) {
const gateway = segment.gateway;
if (!gateway) continue;
const existing = plans.find(
(plan) =>
plan.outside === gateway.to &&
plan.outsideAddresses.some((address) => gateway.address.includes(address)),
);
if (existing) {
existing.inside.push(segmentName);
// The union, so a gateway declared with a v6 address on only one of the segments it
// serves still carries it. The declarations must otherwise agree — validate refuses
// the case where they do not, so there is nothing to reconcile here.
for (const address of gateway.address) {
if (!existing.outsideAddresses.includes(address)) existing.outsideAddresses.push(address);
}
continue;
}
plans.push({
name: `mlab-${instanceId}-gw${plans.length}`,
inside: [segmentName],
outside: gateway.to,
outsideAddresses: [...gateway.address],
nat: gateway.nat,
forwardable: gateway.forwardable,
mappingTtl: gateway.mappingTtl,
});
}
return plans;
}
/** "120s" / "2m" / "90" → seconds. */
export function ttlSeconds(text: string | undefined): number | undefined {
if (!text) return undefined;
const match = /^(\d+)\s*([smh]?)$/.exec(text.trim());
if (!match) return undefined;
const value = Number(match[1]);
return match[2] === "m" ? value * 60 : match[2] === "h" ? value * 3600 : value;
}
function withPrefix(scenario: Scenario, segment: string, address: string): string {
const wantV6 = address.includes(":");
for (const range of scenario.segments[segment]?.cidr ?? []) {
const slash = range.lastIndexOf("/");
if (slash === -1) continue;
if (range.slice(0, slash).includes(":") === wantV6) return `${address}${range.slice(slash)}`;
}
return address;
}
/** The address a machine holds on a segment, for DNAT targets and as an inside gateway. */
function addressOn(scenario: Scenario, machine: string, segment: string, family: Family): string | null {
const spec = scenario.machines[machine];
if (!spec || spec.at === "detached") return null;
for (const attachment of spec.at) {
if (attachment.segment !== segment) continue;
for (const address of attachment.address) {
if ((family === "v6") === address.includes(":")) return address;
}
}
return null;
}
/**
* The router's own address on an inside segment: the first host address of that range.
*
* Chosen rather than declared because a scenario has nothing to say about it — the
* declaration describes what the world sees the network as, and the inside address is an
* implementation detail of the machine serving it.
*/
function insideAddress(scenario: Scenario, segment: string, family: Family): string | null {
for (const range of scenario.segments[segment]?.cidr ?? []) {
const slash = range.lastIndexOf("/");
if (slash === -1) continue;
const base = range.slice(0, slash);
const isV6 = base.includes(":");
if (isV6 !== (family === "v6")) continue;
if (isV6) return `${base.replace(/::$/, "::")}1${range.slice(slash)}`.replace("::1/", "::1/");
const octets = base.split(".");
octets[3] = "1";
return `${octets.join(".")}${range.slice(slash)}`;
}
return null;
}
/**
* Wire the public segments together.
*
* The internet is not a network — it is unrelated networks that route to each other, many
* hops apart with no shared broadcast domain. So public segments are separate links joined
* by a router, never bridged: bridging them would make ARP adjacency, non-decrementing TTL
* and crossing multicast true in the lab and false in production, and the mesh has already
* been bitten by multicast name resolution.
*
* One transit router, an interface on every public segment, forwarding and no translation.
* It is the closest thing the lab has to "the internet", and it is deliberately dumb.
*/
export async function raiseTransit(
scenario: Scenario,
instanceId: string,
log: (message: string) => void = () => {},
): Promise<string | null> {
const publicSegments = Object.entries(scenario.segments)
.filter(([, segment]) => segment.kind === "public")
.map(([name]) => name);
// One public network needs no transit: everything on it is already adjacent.
if (publicSegments.length < 2) return null;
const name = `mlab-${instanceId}-transit`;
if (!(await succeeds(["config", "show", name], 15_000))) {
await incus([
"init", ROUTER_IMAGE, name,
"-c", `user.mesh-lab.instance=${instanceId}`,
"-c", "user.mesh-lab.machine=transit",
"-c", `user.mesh-lab.transit=${publicSegments.join(",")}`,
], 300_000);
await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
for (const [index, segment] of publicSegments.entries()) {
await incus([
"config", "device", "add", name, `eth${index}`, "nic",
"nictype=bridged",
`parent=${networkName(instanceId, segment)}`,
`hwaddr=${macFor(instanceId, "transit", index)}`,
]);
}
}
await succeeds(["start", name], 60_000);
await waitUntilUsable(name, 120, () => {});
for (const [index, segment] of publicSegments.entries()) {
const device = `eth${index}`;
await sh(name, `ip link set ${device} up`);
for (const cidr of scenario.segments[segment]?.cidr ?? []) {
const address = transitAddress(cidr);
if (address) await sh(name, `ip addr replace ${address} dev ${device}`);
}
const mtu = scenario.segments[segment]?.mtu;
if (mtu) await sh(name, `ip link set ${device} mtu ${mtu}`);
}
await sh(
name,
"sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
);
log(` transit router across ${publicSegments.join(", ")}`);
return name;
}
export async function raiseRouters(
scenario: Scenario,
instanceId: string,
plans: RouterPlan[],
log: (message: string) => void = () => {},
): Promise<string[]> {
const created: string[] = [];
if (plans.length > 0) await ensureRouterImage(log);
for (const plan of plans) {
if (!(await succeeds(["config", "show", plan.name], 15_000))) {
await incus([
"init", ROUTER_IMAGE, plan.name,
"-c", `user.mesh-lab.instance=${instanceId}`,
// Tagged as a machine as well as a router: destroy finds an instance's resources
// with one query, and a router that only carried `router=` was left behind — which
// then held its networks open, so `destroy` reported removing zero segments.
"-c", `user.mesh-lab.machine=${routerMachineName(plan)}`,
"-c", `user.mesh-lab.router=${plan.inside.join(",")}`,
// `outside` is structural — it is which link eth0 is on, true the moment the device
// is added. The gateway's *behaviour* is not recorded here; see configureRouter.
"-c", `user.mesh-lab.outside=${plan.outside}`,
], 300_000);
await succeeds(["config", "device", "remove", plan.name, "eth0"], 15_000);
// eth0 faces outward, then one interface per segment behind it.
const links = [plan.outside, ...plan.inside];
for (const [index, segment] of links.entries()) {
await incus([
"config", "device", "add", plan.name, `eth${index}`, "nic",
"nictype=bridged",
`parent=${networkName(instanceId, segment)}`,
`hwaddr=${macFor(instanceId, `gw-${plan.name}`, index)}`,
]);
}
}
await succeeds(["start", plan.name], 60_000);
created.push(plan.name);
log(` router ${plan.inside.join("+")} → ${plan.outside}`);
}
for (const name of created) {
await waitUntilUsable(name, 120, () => {});
}
for (const plan of plans) {
await configureRouter(scenario, plan, log);
}
return created;
}
async function sh(name: string, script: string, timeoutMs = 60_000): Promise<void> {
await incus(["exec", name, "--", "sh", "-c", script], timeoutMs);
}
async function configureRouter(
scenario: Scenario,
plan: RouterPlan,
log: (message: string) => void,
): Promise<void> {
// Addresses: eth0 outside, then one per inside segment.
const links: { device: string; segment: string; addresses: string[] }[] = [
{
device: "eth0",
segment: plan.outside,
addresses: plan.outsideAddresses.map((a) => withPrefix(scenario, plan.outside, a)),
},
];
for (const [index, segment] of plan.inside.entries()) {
const addresses: string[] = [];
for (const family of ["v4", "v6"] as Family[]) {
const address = insideAddress(scenario, segment, family);
if (address) addresses.push(address);
}
links.push({ device: `eth${index + 1}`, segment, addresses });
}
for (const link of links) {
// Up first: an address on a down interface is accepted and then not used.
await sh(plan.name, `ip link set ${link.device} up`);
for (const address of link.addresses) {
// `replace` rather than `add`, so re-running is safe and a real failure still fails.
await sh(plan.name, `ip addr replace ${address} dev ${link.device}`);
}
const mtu = scenario.segments[link.segment]?.mtu;
if (mtu) await sh(plan.name, `ip link set ${link.device} mtu ${mtu}`);
}
await sh(
plan.name,
"sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
);
// A gateway reaches other public networks the way anything does: through transit. Without
// this it can only reach its own outside segment, and every scenario with more than one
// public network becomes a set of islands.
for (const cidr of scenario.segments[plan.outside]?.cidr ?? []) {
const via = transitAddress(cidr);
if (!via) continue;
const gateway = via.slice(0, via.lastIndexOf("/"));
const family = gateway.includes(":") ? "-6" : "-4";
await sh(plan.name, `ip ${family} route replace default via ${gateway} dev eth0 2>/dev/null || true`);
}
const ttl = ttlSeconds(plan.mappingTtl);
if (ttl !== undefined) {
// What makes keepalive behaviour testable rather than hoped for: a connection held
// through NAT without refreshing dies when the mapping does.
//
// Read back rather than assumed. These sysctls are not present on every kernel, and a
// scenario that declared an expiring mapping and silently got a permanent one would be
// the fault this lab exists to catch.
await sh(
plan.name,
`sysctl -w net.netfilter.nf_conntrack_udp_timeout=${ttl} >/dev/null 2>&1; ` +
`sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=${ttl} >/dev/null 2>&1; true`,
);
const readback = await incus(
["exec", plan.name, "--", "sh", "-c",
"cat /proc/sys/net/netfilter/nf_conntrack_udp_timeout 2>/dev/null || echo missing"],
20_000,
);
if (readback.stdout.trim() !== String(ttl)) {
throw new Error(
`${plan.name}: mapping_ttl of ${plan.mappingTtl} was declared but conntrack reports ` +
`'${readback.stdout.trim()}'. The scenario would silently have permanent mappings.`,
);
}
}
await applyRules(scenario, plan);
// Recorded last, and only here. Everything above either read itself back or threw, so a
// gateway carrying these tags is one that demonstrably does these things. Written at
// `init` they would have been a restatement of the request — and a raise that failed
// half way leaves its wreckage standing on purpose, so a picture of that wreckage would
// have badged translation the router was never configured to do.
const recorded = [
`user.mesh-lab.nat=${plan.nat.join(",")}`,
`user.mesh-lab.forwardable=${plan.forwardable}`,
...(ttl === undefined ? [] : [`user.mesh-lab.mapping-ttl=${ttl}`]),
];
for (const entry of recorded) {
const at = entry.indexOf("=");
await succeeds(["config", "set", plan.name, entry.slice(0, at), entry.slice(at + 1)], 20_000);
}
log(` ${plan.name}: nat=${plan.nat.join(",") || "none"} forwardable=${plan.forwardable}${ttl ? ` ttl=${ttl}s` : ""}`);
}
/** One ruleset per router, written whole — partial rule edits drift, a whole file does not. */
async function applyRules(scenario: Scenario, plan: RouterPlan): Promise<void> {
const parts: string[] = ["flush ruleset"];
for (const family of plan.nat) {
const table = family === "v4" ? "ip" : "ip6";
parts.push(
`table ${table} nat {`,
` chain postrouting { type nat hook postrouting priority srcnat; policy accept;`,
` oifname "eth0" masquerade`,
` }`,
` chain prerouting { type nat hook prerouting priority dstnat; policy accept;`,
);
if (plan.forwardable) {
for (const [machine, spec] of Object.entries(scenario.machines)) {
for (const publication of spec.published ?? []) {
if (!plan.inside.includes(publication.on)) continue;
const target = addressOn(scenario, machine, publication.on, family);
if (!target) continue;
const destination = family === "v6" ? `[${target}]` : target;
parts.push(
` iifname "eth0" tcp dport ${publication.port} dnat to ${destination}:${publication.port}`,
);
}
}
}
parts.push(` }`, `}`);
}
// Filtering: unsolicited inbound, and policy between segments the router serves.
const filterLines: string[] = [];
if (!plan.forwardable) {
// A gateway you do not control: outbound works, nothing initiates inward. That is the
// constraint being reproduced, not an implementation limit.
filterLines.push(` iifname "eth0" ct state new drop`);
}
for (const rule of scenario.policy ?? []) {
if (rule.allow) continue;
const fromIndex = plan.inside.indexOf(rule.from);
const toIndex = plan.inside.indexOf(rule.to);
if (fromIndex === -1 || toIndex === -1) continue;
filterLines.push(` iifname "eth${fromIndex + 1}" oifname "eth${toIndex + 1}" drop`);
}
if (filterLines.length > 0) {
parts.push(
`table inet filter {`,
` chain forward { type filter hook forward priority filter; policy accept;`,
` ct state established,related accept`,
...filterLines,
` }`,
`}`,
);
}
const ruleset = parts.join("\n");
await sh(
plan.name,
`cat > /tmp/mlab.nft <<'MLABNFT'\n${ruleset}\nMLABNFT\nnft -f /tmp/mlab.nft`,
60_000,
);
}