novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Transit, host firewalls, and the whole topology raising

Browse Source 
The full topology now raises: four machines, three routers, a transit
router, six segments, in 35 seconds. Everything the declaration model can
express except `place`, which is refused because the node host it would
place does not exist yet.

Transit was a real gap, not a bug. The design says public networks are
unrelated and routed to each other, never bridged — and I built the
segments and never built the thing that routes between them, so three
public networks were islands and nothing crossed. A transit router now
holds an interface on every public segment, forwarding and no translation:
the closest thing the lab has to the internet, deliberately dumb.

Proven rather than asserted, by ping TTL across the raised topology:

  within one segment                     ttl=64   no hops
  across two unrelated public networks   ttl=62   gateway + transit
  multicast between public networks      0 replies

A flat internet would have shown ttl=64 and answered multicast — which
would let a node discover a peer it could never reach in production, and
report success. That is the fault the as-is layer records the mesh already
hitting with multicast name resolution.

inbound: deny is implemented as a host firewall on the machine, read back
after applying. A declared refusal that silently did not load leaves the
machine wide open, which looks exactly like a machine that is working.
Established and related traffic is accepted, so a defended machine can
still dial out rather than being a disconnected one.

Verified by running, all of it:

  home -> devices (policy allow)               reachable
  devices -> home (policy deny)                blocked
  behind unforwardable NAT -> out              reachable
  in -> behind unforwardable NAT               unreachable
  inbound: deny, dialling out                  reachable
  reaching a machine that denies inbound       refused

The two routers differ exactly as declared: the forwardable one carries the
policy rule and no inbound drop, the unforwardable one carries `ct state
new drop` and no DNAT.
This commit is contained in:
jschoubben
2026-08-24 01:49:30 +02:00
parent a270cd5b02
commit 5d01006eab
9 changed files with 295 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lifecycle/address.ts
+33 -4
View File
@@ -13,9 +13,10 @@
* See novox/hq 02-DECISIONS/0031-the-lab-provides-the-underlay.md
*/
import type { Scenario } from "../declaration/types.ts";
import type { Attachment, Scenario } from "../declaration/types.ts";
import { incus } from "../incus/client.ts";
import { macFor } from "./names.ts";
import { transitAddress } from "./router.ts";
export interface Wire {
device: string;
@@ -120,10 +121,14 @@ export async function applyDefaultRoutes(
const name = machineNames.get(machine);
if (!name) continue;
// A machine behind a gateway routes through it. A machine attached directly to a public
// segment has nowhere to default to, and should not pretend otherwise.
// A machine behind a gateway routes through it. A machine sitting directly on a public
// segment routes through transit instead — otherwise it can reach its own network and
// nothing else, which is not what being on the internet means.
const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway);
if (!behind) continue;
if (!behind) {
await routeViaTransit(scenario, spec, name);
continue;
}
const index = spec.at.indexOf(behind);
for (const range of scenario.segments[behind.segment]?.cidr ?? []) {
@@ -143,3 +148,27 @@ export async function applyDefaultRoutes(
log(` routed ${machine} via its gateway on ${behind.segment}`);
}
}
/** A machine on a public segment reaches the other public networks through transit. */
async function routeViaTransit(
scenario: Scenario,
spec: { at: Attachment[] | "detached" },
name: string,
): Promise<void> {
if (spec.at === "detached") return;
const onPublic = spec.at.find((a) => scenario.segments[a.segment]?.kind === "public");
if (!onPublic) return;
const index = spec.at.indexOf(onPublic);
for (const cidr of scenario.segments[onPublic.segment]?.cidr ?? []) {
const via = transitAddress(cidr);
if (!via) continue;
const gateway = via.slice(0, via.lastIndexOf("/"));
const family = gateway.includes(":") ? "-6" : "-4";
await incus(
["exec", name, "--", "sh", "-c",
`ip ${family} route replace default via ${gateway} dev $(ip -o link | awk -F': ' 'NR==${index + 2}{print $2}') 2>/dev/null || true`],
30_000,
);
}
}
src/lifecycle/firewall.ts
+59
View File
@@ -0,0 +1,59 @@
/**
* `inbound: deny` — a host firewall on the machine itself.
*
* Distinct from NAT and behaving differently: a machine can be perfectly routable and
* still refuse everything unsolicited, which is the normal state of a v6-addressed
* machine. Without this, v6 addressing would silently imply reachability, and a scenario
* that said a machine refuses traffic would produce one that accepts it.
*
* Established and related traffic is accepted, so the machine can still dial out. That is
* what a host firewall does; a machine that could not reach anything would be reproducing
* a disconnected machine rather than a defended one.
*/
import type { Scenario } from "../declaration/types.ts";
import { incus } from "../incus/client.ts";
const RULESET = `flush ruleset
table inet mlab {
chain input {
type filter hook input priority filter; policy drop;
ct state established,related accept
iif lo accept
ct state invalid drop
}
}
`;
export async function applyHostFirewalls(
scenario: Scenario,
machineNames: Map<string, string>,
log: (message: string) => void = () => {},
): Promise<void> {
for (const [machine, spec] of Object.entries(scenario.machines)) {
if (spec.inbound !== "deny") continue;
const name = machineNames.get(machine);
if (!name) continue;
await incus(
["exec", name, "--", "sh", "-c",
`cat > /tmp/mlab-host.nft <<'MLABNFT'\n${RULESET}MLABNFT\nnft -f /tmp/mlab-host.nft`],
60_000,
);
// Read back. A declared refusal that silently did not apply is the fault this lab
// exists to catch, and a ruleset that failed to load leaves the machine wide open —
// which looks exactly like a machine that is working.
const check = await incus(
["exec", name, "--", "sh", "-c", "nft list table inet mlab >/dev/null 2>&1 && echo present || echo absent"],
20_000,
);
if (check.stdout.trim() !== "present") {
throw new Error(
`${machine}: inbound: deny was declared but the ruleset is not loaded — the machine ` +
`would accept traffic the scenario says it refuses`,
);
}
log(` ${machine} refuses unsolicited inbound`);
}
}
src/lifecycle/raise.ts
+12 -1
View File
@@ -20,7 +20,8 @@ import { machineName, macFor, networkName, newInstanceId } from "./names.ts";
import { waitUntilAllUsable } from "./ready.ts";
import { applyAddresses, applyDefaultRoutes } from "./address.ts";
import { assertSupported } from "./supported.ts";
import { planRouters, raiseRouters } from "./router.ts";
import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
import { applyHostFirewalls } from "./firewall.ts";
/** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
const COW_DRIVERS = ["btrfs", "zfs"];
@@ -196,12 +197,22 @@ export async function raise(
step = "applying declared addresses";
await applyAddresses(scenario, instanceId, byMachine, log);
// Transit first: a gateway's default route points at it, so it has to exist.
step = "wiring the public networks together";
const transit = await raiseTransit(scenario, instanceId, log);
step = "raising routers";
const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
if (transit) routers.push(transit);
step = "routing machines through their gateways";
await applyDefaultRoutes(scenario, byMachine, log);
// Last: a machine that refuses inbound must still have been reachable while the lab
// was configuring it.
step = "applying host firewalls";
await applyHostFirewalls(scenario, byMachine, log);
return {
instanceId,
scenario: scenario.scenario,
src/lifecycle/router.ts
+92
View File
@@ -91,6 +91,23 @@ export async function ensureRouterImage(log: (message: string) => void = () => {
log(` router image ready`);
}
/**
* The address the transit router holds on a public segment: the last usable host address.
*
* Chosen rather than declared, like a gateway's inside address — a scenario has nothing to
* say about the internet's own routers, only about the networks they connect.
*/
export function transitAddress(cidr: string): string | null {
const slash = cidr.lastIndexOf("/");
if (slash === -1) return null;
const base = cidr.slice(0, slash);
const prefix = cidr.slice(slash);
if (base.includes(":")) return `${base.replace(/::$/, "")}::fffe${prefix}`;
const octets = base.split(".");
octets[3] = "254";
return `${octets.join(".")}${prefix}`;
}
/** The name a router answers to in `list` and `exec` — scenery, but addressable. */
export function routerMachineName(plan: RouterPlan): string {
return `gw-${plan.inside.join("-")}`;
@@ -197,6 +214,70 @@ function insideAddress(scenario: Scenario, segment: string, family: Family): str
return null;
}
/**
* Wire the public segments together.
*
* The internet is not a network — it is unrelated networks that route to each other, many
* hops apart with no shared broadcast domain. So public segments are separate links joined
* by a router, never bridged: bridging them would make ARP adjacency, non-decrementing TTL
* and crossing multicast true in the lab and false in production, and the mesh has already
* been bitten by multicast name resolution.
*
* One transit router, an interface on every public segment, forwarding and no translation.
* It is the closest thing the lab has to "the internet", and it is deliberately dumb.
*/
export async function raiseTransit(
scenario: Scenario,
instanceId: string,
log: (message: string) => void = () => {},
): Promise<string | null> {
const publicSegments = Object.entries(scenario.segments)
.filter(([, segment]) => segment.kind === "public")
.map(([name]) => name);
// One public network needs no transit: everything on it is already adjacent.
if (publicSegments.length < 2) return null;
const name = `mlab-${instanceId}-transit`;
if (!(await succeeds(["config", "show", name], 15_000))) {
await incus([
"init", ROUTER_IMAGE, name,
"-c", `user.mesh-lab.instance=${instanceId}`,
"-c", "user.mesh-lab.machine=transit",
"-c", `user.mesh-lab.transit=${publicSegments.join(",")}`,
], 300_000);
await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
for (const [index, segment] of publicSegments.entries()) {
await incus([
"config", "device", "add", name, `eth${index}`, "nic",
"nictype=bridged",
`parent=${networkName(instanceId, segment)}`,
`hwaddr=${macFor(instanceId, "transit", index)}`,
]);
}
}
await succeeds(["start", name], 60_000);
await waitUntilUsable(name, 120, () => {});
for (const [index, segment] of publicSegments.entries()) {
const device = `eth${index}`;
await sh(name, `ip link set ${device} up`);
for (const cidr of scenario.segments[segment]?.cidr ?? []) {
const address = transitAddress(cidr);
if (address) await sh(name, `ip addr replace ${address} dev ${device}`);
}
const mtu = scenario.segments[segment]?.mtu;
if (mtu) await sh(name, `ip link set ${device} mtu ${mtu}`);
}
await sh(
name,
"sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
);
log(` transit router across ${publicSegments.join(", ")}`);
return name;
}
export async function raiseRouters(
scenario: Scenario,
instanceId: string,
@@ -289,6 +370,17 @@ async function configureRouter(
"sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
);
// A gateway reaches other public networks the way anything does: through transit. Without
// this it can only reach its own outside segment, and every scenario with more than one
// public network becomes a set of islands.
for (const cidr of scenario.segments[plan.outside]?.cidr ?? []) {
const via = transitAddress(cidr);
if (!via) continue;
const gateway = via.slice(0, via.lastIndexOf("/"));
const family = gateway.includes(":") ? "-6" : "-4";
await sh(plan.name, `ip ${family} route replace default via ${gateway} dev eth0 2>/dev/null || true`);
}
const ttl = ttlSeconds(plan.mappingTtl);
if (ttl !== undefined) {
// What makes keepalive behaviour testable rather than hoped for: a connection held
src/lifecycle/supported.ts
-10
View File
@@ -33,16 +33,6 @@ export class UnsupportedError extends Error {
export function assertSupported(scenario: Scenario): void {
const missing: string[] = [];
const inbound = Object.entries(scenario.machines)
.filter(([, machine]) => machine.inbound === "deny")
.map(([name]) => name);
if (inbound.length > 0) {
missing.push(
`inbound: deny (machines: ${inbound.join(", ")}) — no host firewall is configured, so ` +
`these machines would accept traffic the scenario says they refuse`,
);
}
if (scenario.place && Object.keys(scenario.place).length > 0) {
missing.push(
"place — nothing is placed inside the machines yet; they are raised bare",