diff --git a/README.md b/README.md index 7302f2f..43a78ca 100644 --- a/README.md +++ b/README.md @@ -110,10 +110,11 @@ than ignored: | gateways, NAT, masquerade | **works** | | `published:` ports (DNAT through the gateway's address) | **works** | | `mapping_ttl:` (conntrack timeout) | **works**, and verified after setting — a declared expiry that silently did not apply would be the fault this catches | -| `forwardable: false` | implemented, **not yet verified by running** | -| `policy:` between segments | implemented, **not yet verified by running** | -| `inbound: deny` | **refused at raise** | -| `place:` | **refused at raise** | +| `forwardable: false` | **works** — outbound only, no DNAT, unsolicited inbound dropped | +| `policy:` between segments | **works**, asymmetric | +| `inbound: deny` | **works** — host firewall, read back after applying | +| several public networks, routed not bridged | **works** — a transit router, never a shared bridge | +| `place:` | **refused at raise** — the node host it would place does not exist yet | `raise` refuses a scenario declaring anything in the lower half, naming every gap. It does not raise a mesh that silently lacks what it declared — that is the fault this lab exists to catch @@ -138,12 +139,29 @@ something under test (`novox/hq` ADR 0033). gateway, reached from a machine on a routable address: ``` -home-server -> anchor 0% loss, through masquerade -anchor -> 192.168.1.135 (private, direct) unreachable ✓ -anchor -> 192.0.2.50:8080 (the GATEWAY) HTTP 200 +home-server -> anchor 0% loss, through masquerade +anchor -> 192.168.1.135 (private, direct) unreachable ✓ +anchor -> 192.0.2.50:8080 (the GATEWAY) HTTP 200 +home -> devices (policy allow) reachable ✓ +devices -> home (policy deny) blocked ✓ +roamer behind unforwardable NAT -> anchor reachable ✓ (outbound only) +anchor -> roamer unreachable ✓ +workstation with inbound: deny, dialling out reachable ✓ (defended, not disconnected) +home-server -> workstation refused ✓ ``` -The last line is the case research 004 says only exists in production. +The third line is the case research 004 says only exists in production. + +**Routed, never bridged**, proven rather than asserted — ping TTL across the full topology: + +``` +within one segment ttl=64 no hops +across two unrelated public networks ttl=62 gateway + transit +multicast between public networks 0 replies +``` + +A flat "internet" would have shown ttl=64 and answered multicast, which would have let a node +discover a peer it could never reach in production — and report success. Machines boot concurrently, so a second machine costs seconds rather than doubling the wait. Nearly all of the remaining time is boot, which cannot be avoided. diff --git a/scenarios/segmented-and-unforwardable.yml b/scenarios/segmented-and-unforwardable.yml new file mode 100644 index 0000000..b7cf4b7 --- /dev/null +++ b/scenarios/segmented-and-unforwardable.yml @@ -0,0 +1,65 @@ +# Two things production has and a flat lab cannot show. +# +# `devices` and `home` sit behind ONE router — identical gateway declarations — with a +# policy allowing home→devices and denying the reverse. That is an ordinary segmented +# household router, and the asymmetry is the normal case. +# +# `cafe` sits behind a gateway we do not control. Outbound works; nothing initiates +# inward, and nothing can be published there at all. +scenario: segmented-and-unforwardable + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + + home: + kind: private + cidr: [192.168.1.0/24] + gateway: + to: hosting + address: [192.0.2.50] + nat: [v4] + forwardable: true + mapping_ttl: 120s + + devices: + kind: private + cidr: [192.168.30.0/24] + gateway: + to: hosting + address: [192.0.2.50] # identical → the SAME router + nat: [v4] + forwardable: true + mapping_ttl: 120s + + cafe: + kind: private + cidr: [10.50.0.0/16] + gateway: + to: hosting + address: [192.0.2.80] + nat: [v4] + forwardable: false # carrier-grade NAT, or simply not ours + mapping_ttl: 30s + +policy: + - { from: devices, to: home, allow: false } + - { from: home, to: devices, allow: true } + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + + home-server: + at: { segment: home, address: [192.168.1.135] } + inbound: allow + + thermostat: + at: { segment: devices, address: [192.168.30.20] } + inbound: allow + + roamer: + at: { segment: cafe, address: [10.50.3.23] } + inbound: allow diff --git a/scenarios/the-ordinary-shape.yml b/scenarios/the-ordinary-shape.yml index 04e7022..3277605 100644 --- a/scenarios/the-ordinary-shape.yml +++ b/scenarios/the-ordinary-shape.yml @@ -74,8 +74,8 @@ machines: at: { segment: home, address: [192.168.1.98, "2001:db8:b:1::98"] } inbound: deny -place: - all: [host] - anchor: [substrate] +# No `place:` yet. The node host it would place does not exist — this lab is being built to +# develop it, and the lab refuses declarations it cannot materialise rather than raising a +# mesh that silently lacks them. snapshot: raised diff --git a/src/lifecycle/address.ts b/src/lifecycle/address.ts index 44994a6..21bc5a3 100644 --- a/src/lifecycle/address.ts +++ b/src/lifecycle/address.ts @@ -13,9 +13,10 @@ * See novox/hq 02-DECISIONS/0031-the-lab-provides-the-underlay.md */ -import type { Scenario } from "../declaration/types.ts"; +import type { Attachment, Scenario } from "../declaration/types.ts"; import { incus } from "../incus/client.ts"; import { macFor } from "./names.ts"; +import { transitAddress } from "./router.ts"; export interface Wire { device: string; @@ -120,10 +121,14 @@ export async function applyDefaultRoutes( const name = machineNames.get(machine); if (!name) continue; - // A machine behind a gateway routes through it. A machine attached directly to a public - // segment has nowhere to default to, and should not pretend otherwise. + // A machine behind a gateway routes through it. A machine sitting directly on a public + // segment routes through transit instead — otherwise it can reach its own network and + // nothing else, which is not what being on the internet means. const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway); - if (!behind) continue; + if (!behind) { + await routeViaTransit(scenario, spec, name); + continue; + } const index = spec.at.indexOf(behind); for (const range of scenario.segments[behind.segment]?.cidr ?? []) { @@ -143,3 +148,27 @@ export async function applyDefaultRoutes( log(` routed ${machine} via its gateway on ${behind.segment}`); } } + +/** A machine on a public segment reaches the other public networks through transit. */ +async function routeViaTransit( + scenario: Scenario, + spec: { at: Attachment[] | "detached" }, + name: string, +): Promise { + if (spec.at === "detached") return; + const onPublic = spec.at.find((a) => scenario.segments[a.segment]?.kind === "public"); + if (!onPublic) return; + + const index = spec.at.indexOf(onPublic); + for (const cidr of scenario.segments[onPublic.segment]?.cidr ?? []) { + const via = transitAddress(cidr); + if (!via) continue; + const gateway = via.slice(0, via.lastIndexOf("/")); + const family = gateway.includes(":") ? "-6" : "-4"; + await incus( + ["exec", name, "--", "sh", "-c", + `ip ${family} route replace default via ${gateway} dev $(ip -o link | awk -F': ' 'NR==${index + 2}{print $2}') 2>/dev/null || true`], + 30_000, + ); + } +} diff --git a/src/lifecycle/firewall.ts b/src/lifecycle/firewall.ts new file mode 100644 index 0000000..5da9b80 --- /dev/null +++ b/src/lifecycle/firewall.ts @@ -0,0 +1,59 @@ +/** + * `inbound: deny` — a host firewall on the machine itself. + * + * Distinct from NAT and behaving differently: a machine can be perfectly routable and + * still refuse everything unsolicited, which is the normal state of a v6-addressed + * machine. Without this, v6 addressing would silently imply reachability, and a scenario + * that said a machine refuses traffic would produce one that accepts it. + * + * Established and related traffic is accepted, so the machine can still dial out. That is + * what a host firewall does; a machine that could not reach anything would be reproducing + * a disconnected machine rather than a defended one. + */ + +import type { Scenario } from "../declaration/types.ts"; +import { incus } from "../incus/client.ts"; + +const RULESET = `flush ruleset +table inet mlab { + chain input { + type filter hook input priority filter; policy drop; + ct state established,related accept + iif lo accept + ct state invalid drop + } +} +`; + +export async function applyHostFirewalls( + scenario: Scenario, + machineNames: Map, + log: (message: string) => void = () => {}, +): Promise { + for (const [machine, spec] of Object.entries(scenario.machines)) { + if (spec.inbound !== "deny") continue; + const name = machineNames.get(machine); + if (!name) continue; + + await incus( + ["exec", name, "--", "sh", "-c", + `cat > /tmp/mlab-host.nft <<'MLABNFT'\n${RULESET}MLABNFT\nnft -f /tmp/mlab-host.nft`], + 60_000, + ); + + // Read back. A declared refusal that silently did not apply is the fault this lab + // exists to catch, and a ruleset that failed to load leaves the machine wide open — + // which looks exactly like a machine that is working. + const check = await incus( + ["exec", name, "--", "sh", "-c", "nft list table inet mlab >/dev/null 2>&1 && echo present || echo absent"], + 20_000, + ); + if (check.stdout.trim() !== "present") { + throw new Error( + `${machine}: inbound: deny was declared but the ruleset is not loaded — the machine ` + + `would accept traffic the scenario says it refuses`, + ); + } + log(` ${machine} refuses unsolicited inbound`); + } +} diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index e2af849..5c1125c 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -20,7 +20,8 @@ import { machineName, macFor, networkName, newInstanceId } from "./names.ts"; import { waitUntilAllUsable } from "./ready.ts"; import { applyAddresses, applyDefaultRoutes } from "./address.ts"; import { assertSupported } from "./supported.ts"; -import { planRouters, raiseRouters } from "./router.ts"; +import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; +import { applyHostFirewalls } from "./firewall.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ const COW_DRIVERS = ["btrfs", "zfs"]; @@ -196,12 +197,22 @@ export async function raise( step = "applying declared addresses"; await applyAddresses(scenario, instanceId, byMachine, log); + // Transit first: a gateway's default route points at it, so it has to exist. + step = "wiring the public networks together"; + const transit = await raiseTransit(scenario, instanceId, log); + step = "raising routers"; const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log); + if (transit) routers.push(transit); step = "routing machines through their gateways"; await applyDefaultRoutes(scenario, byMachine, log); + // Last: a machine that refuses inbound must still have been reachable while the lab + // was configuring it. + step = "applying host firewalls"; + await applyHostFirewalls(scenario, byMachine, log); + return { instanceId, scenario: scenario.scenario, diff --git a/src/lifecycle/router.ts b/src/lifecycle/router.ts index 89f22d5..b39aa51 100644 --- a/src/lifecycle/router.ts +++ b/src/lifecycle/router.ts @@ -91,6 +91,23 @@ export async function ensureRouterImage(log: (message: string) => void = () => { log(` router image ready`); } +/** + * The address the transit router holds on a public segment: the last usable host address. + * + * Chosen rather than declared, like a gateway's inside address — a scenario has nothing to + * say about the internet's own routers, only about the networks they connect. + */ +export function transitAddress(cidr: string): string | null { + const slash = cidr.lastIndexOf("/"); + if (slash === -1) return null; + const base = cidr.slice(0, slash); + const prefix = cidr.slice(slash); + if (base.includes(":")) return `${base.replace(/::$/, "")}::fffe${prefix}`; + const octets = base.split("."); + octets[3] = "254"; + return `${octets.join(".")}${prefix}`; +} + /** The name a router answers to in `list` and `exec` — scenery, but addressable. */ export function routerMachineName(plan: RouterPlan): string { return `gw-${plan.inside.join("-")}`; @@ -197,6 +214,70 @@ function insideAddress(scenario: Scenario, segment: string, family: Family): str return null; } +/** + * Wire the public segments together. + * + * The internet is not a network — it is unrelated networks that route to each other, many + * hops apart with no shared broadcast domain. So public segments are separate links joined + * by a router, never bridged: bridging them would make ARP adjacency, non-decrementing TTL + * and crossing multicast true in the lab and false in production, and the mesh has already + * been bitten by multicast name resolution. + * + * One transit router, an interface on every public segment, forwarding and no translation. + * It is the closest thing the lab has to "the internet", and it is deliberately dumb. + */ +export async function raiseTransit( + scenario: Scenario, + instanceId: string, + log: (message: string) => void = () => {}, +): Promise { + const publicSegments = Object.entries(scenario.segments) + .filter(([, segment]) => segment.kind === "public") + .map(([name]) => name); + + // One public network needs no transit: everything on it is already adjacent. + if (publicSegments.length < 2) return null; + + const name = `mlab-${instanceId}-transit`; + if (!(await succeeds(["config", "show", name], 15_000))) { + await incus([ + "init", ROUTER_IMAGE, name, + "-c", `user.mesh-lab.instance=${instanceId}`, + "-c", "user.mesh-lab.machine=transit", + "-c", `user.mesh-lab.transit=${publicSegments.join(",")}`, + ], 300_000); + await succeeds(["config", "device", "remove", name, "eth0"], 15_000); + for (const [index, segment] of publicSegments.entries()) { + await incus([ + "config", "device", "add", name, `eth${index}`, "nic", + "nictype=bridged", + `parent=${networkName(instanceId, segment)}`, + `hwaddr=${macFor(instanceId, "transit", index)}`, + ]); + } + } + await succeeds(["start", name], 60_000); + await waitUntilUsable(name, 120, () => {}); + + for (const [index, segment] of publicSegments.entries()) { + const device = `eth${index}`; + await sh(name, `ip link set ${device} up`); + for (const cidr of scenario.segments[segment]?.cidr ?? []) { + const address = transitAddress(cidr); + if (address) await sh(name, `ip addr replace ${address} dev ${device}`); + } + const mtu = scenario.segments[segment]?.mtu; + if (mtu) await sh(name, `ip link set ${device} mtu ${mtu}`); + } + + await sh( + name, + "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null", + ); + log(` transit router across ${publicSegments.join(", ")}`); + return name; +} + export async function raiseRouters( scenario: Scenario, instanceId: string, @@ -289,6 +370,17 @@ async function configureRouter( "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null", ); + // A gateway reaches other public networks the way anything does: through transit. Without + // this it can only reach its own outside segment, and every scenario with more than one + // public network becomes a set of islands. + for (const cidr of scenario.segments[plan.outside]?.cidr ?? []) { + const via = transitAddress(cidr); + if (!via) continue; + const gateway = via.slice(0, via.lastIndexOf("/")); + const family = gateway.includes(":") ? "-6" : "-4"; + await sh(plan.name, `ip ${family} route replace default via ${gateway} dev eth0 2>/dev/null || true`); + } + const ttl = ttlSeconds(plan.mappingTtl); if (ttl !== undefined) { // What makes keepalive behaviour testable rather than hoped for: a connection held diff --git a/src/lifecycle/supported.ts b/src/lifecycle/supported.ts index ee6bbc7..c724410 100644 --- a/src/lifecycle/supported.ts +++ b/src/lifecycle/supported.ts @@ -33,16 +33,6 @@ export class UnsupportedError extends Error { export function assertSupported(scenario: Scenario): void { const missing: string[] = []; - const inbound = Object.entries(scenario.machines) - .filter(([, machine]) => machine.inbound === "deny") - .map(([name]) => name); - if (inbound.length > 0) { - missing.push( - `inbound: deny (machines: ${inbound.join(", ")}) — no host firewall is configured, so ` + - `these machines would accept traffic the scenario says they refuse`, - ); - } - if (scenario.place && Object.keys(scenario.place).length > 0) { missing.push( "place — nothing is placed inside the machines yet; they are raised bare", diff --git a/test/supported.test.ts b/test/supported.test.ts index 70ba0a0..4eab53b 100644 --- a/test/supported.test.ts +++ b/test/supported.test.ts @@ -41,11 +41,11 @@ machines: assert.doesNotThrow(() => assertSupported(scenario)); }); -test("inbound: deny is still refused rather than silently absent", () => { +test("inbound: deny is implemented — a host firewall is applied and read back", () => { const scenario = parseScenario(`scenario: x segments: { net: { kind: public, cidr: [192.0.2.0/24] } } machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } }`); - assert.throws(() => assertSupported(scenario), UnsupportedError); + assert.doesNotThrow(() => assertSupported(scenario)); }); test("place is still refused — there is nothing to place yet", () => { @@ -56,16 +56,16 @@ place: { all: [host] }`); assert.throws(() => assertSupported(scenario), UnsupportedError); }); -test("the refusal names every gap, not just the first", () => { +test("the refusal explains what would silently be missing", () => { const scenario = parseScenario(`scenario: x segments: { net: { kind: public, cidr: [192.0.2.0/24] } } -machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } } +machines: { a: { at: { segment: net, address: [192.0.2.1] } } } place: { all: [host] }`); try { assertSupported(scenario); assert.fail("should have refused"); } catch (err) { - assert.equal((err as UnsupportedError).missing.length, 2); + assert.equal((err as UnsupportedError).missing.length, 1); assert.match(err instanceof Error ? err.message : "", /silently lacks them/); } });