The lab raises a mesh, draws it, and now places tier 0 inside it #1
@@ -107,9 +107,11 @@ than ignored:
|
||||
| declared addresses, both families | **works** |
|
||||
| segment MTU | **works** |
|
||||
| raise · exec · snapshot · restore · destroy · list | **works** |
|
||||
| gateways, NAT, forwarding | **refused at raise** |
|
||||
| `published:` ports | **refused at raise** |
|
||||
| `policy:` between segments | **refused at raise** |
|
||||
| gateways, NAT, masquerade | **works** |
|
||||
| `published:` ports (DNAT through the gateway's address) | **works** |
|
||||
| `mapping_ttl:` (conntrack timeout) | **works**, and verified after setting — a declared expiry that silently did not apply would be the fault this catches |
|
||||
| `forwardable: false` | implemented, **not yet verified by running** |
|
||||
| `policy:` between segments | implemented, **not yet verified by running** |
|
||||
| `inbound: deny` | **refused at raise** |
|
||||
| `place:` | **refused at raise** |
|
||||
|
||||
@@ -123,11 +125,25 @@ it is the topology being built toward, and the tool says exactly what is missing
|
||||
|
||||
## Measured on a workstation
|
||||
|
||||
| | one machine | two machines |
|
||||
|---|---|---|
|
||||
| raise, to usable | 12.5 s | 14.6 s |
|
||||
| snapshot | 0.14 s | 0.28 s |
|
||||
| restore, to usable again | 10.5 s | 11.6 s |
|
||||
| | one machine | two machines | two machines + a router |
|
||||
|---|---|---|---|
|
||||
| raise, to usable | 12.5 s | 14.6 s | 32 s |
|
||||
| snapshot | 0.14 s | 0.28 s | — |
|
||||
| restore, to usable again | 10.5 s | 11.6 s | — |
|
||||
|
||||
A router adds seconds, not a boot: it is a container, because it is scenery rather than
|
||||
something under test (`novox/hq` ADR 0033).
|
||||
|
||||
**Verified by running**, not asserted — a machine at `192.168.1.135` behind a household
|
||||
gateway, reached from a machine on a routable address:
|
||||
|
||||
```
|
||||
home-server -> anchor 0% loss, through masquerade
|
||||
anchor -> 192.168.1.135 (private, direct) unreachable ✓
|
||||
anchor -> 192.0.2.50:8080 (the GATEWAY) HTTP 200
|
||||
```
|
||||
|
||||
The last line is the case research 004 says only exists in production.
|
||||
|
||||
Machines boot concurrently, so a second machine costs seconds rather than doubling the wait.
|
||||
Nearly all of the remaining time is boot, which cannot be avoided.
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# A machine on a routable address, and a machine behind a household connection.
|
||||
#
|
||||
# This is the case that only exists in production today: the second machine is reachable
|
||||
# from the first only through a forwarded port, at the GATEWAY's address, never its own.
|
||||
scenario: behind-nat
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
home:
|
||||
kind: private
|
||||
cidr: [192.168.1.0/24]
|
||||
gateway:
|
||||
to: hosting
|
||||
address: [192.0.2.50] # what the world sees the household as
|
||||
nat: [v4]
|
||||
forwardable: true
|
||||
mapping_ttl: 120s
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
inbound: allow
|
||||
|
||||
home-server: # a dash in the name, on purpose
|
||||
at: { segment: home, address: [192.168.1.135] }
|
||||
published:
|
||||
- { port: 8080, on: home }
|
||||
inbound: allow
|
||||
@@ -98,3 +98,48 @@ function withPrefix(scenario: Scenario, segment: string, address: string): strin
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
/**
|
||||
* Point each machine at the router serving its segment.
|
||||
*
|
||||
* The route is the machine's, not the mesh's — a default route is what a home network hands
|
||||
* out, and a machine that could not reach beyond its own segment would be reproducing the
|
||||
* wrong topology. What the lab still does not supply is the overlay: no peers, no hub, no
|
||||
* names.
|
||||
*
|
||||
* The router's inside address is the first host address of the range, chosen rather than
|
||||
* declared because a scenario has nothing to say about it.
|
||||
*/
|
||||
export async function applyDefaultRoutes(
|
||||
scenario: Scenario,
|
||||
machineNames: Map<string, string>,
|
||||
log: (message: string) => void = () => {},
|
||||
): Promise<void> {
|
||||
for (const [machine, spec] of Object.entries(scenario.machines)) {
|
||||
if (spec.at === "detached") continue;
|
||||
const name = machineNames.get(machine);
|
||||
if (!name) continue;
|
||||
|
||||
// A machine behind a gateway routes through it. A machine attached directly to a public
|
||||
// segment has nowhere to default to, and should not pretend otherwise.
|
||||
const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway);
|
||||
if (!behind) continue;
|
||||
|
||||
const index = spec.at.indexOf(behind);
|
||||
for (const range of scenario.segments[behind.segment]?.cidr ?? []) {
|
||||
const slash = range.lastIndexOf("/");
|
||||
if (slash === -1) continue;
|
||||
const base = range.slice(0, slash);
|
||||
const via = base.includes(":")
|
||||
? `${base.replace(/::$/, "")}::1`
|
||||
: (() => { const o = base.split("."); o[3] = "1"; return o.join("."); })();
|
||||
const family = base.includes(":") ? "-6" : "-4";
|
||||
await incus(
|
||||
["exec", name, "--", "sh", "-c",
|
||||
`ip ${family} route replace default via ${via} dev $(ip -o link | awk -F': ' 'NR==${index + 2}{print $2}') 2>/dev/null || true`],
|
||||
30_000,
|
||||
);
|
||||
}
|
||||
log(` routed ${machine} via its gateway on ${behind.segment}`);
|
||||
}
|
||||
}
|
||||
|
||||
+15
-2
@@ -18,8 +18,9 @@ import type { Scenario } from "../declaration/types.ts";
|
||||
import { incus, incusOk, succeeds, pools, supportedDrivers } from "../incus/client.ts";
|
||||
import { machineName, macFor, networkName, newInstanceId } from "./names.ts";
|
||||
import { waitUntilAllUsable } from "./ready.ts";
|
||||
import { applyAddresses } from "./address.ts";
|
||||
import { applyAddresses, applyDefaultRoutes } from "./address.ts";
|
||||
import { assertSupported } from "./supported.ts";
|
||||
import { planRouters, raiseRouters } from "./router.ts";
|
||||
|
||||
/** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
|
||||
const COW_DRIVERS = ["btrfs", "zfs"];
|
||||
@@ -195,7 +196,19 @@ export async function raise(
|
||||
step = "applying declared addresses";
|
||||
await applyAddresses(scenario, instanceId, byMachine, log);
|
||||
|
||||
return { instanceId, scenario: scenario.scenario, machines: created, networks, pool };
|
||||
step = "raising routers";
|
||||
const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
|
||||
|
||||
step = "routing machines through their gateways";
|
||||
await applyDefaultRoutes(scenario, byMachine, log);
|
||||
|
||||
return {
|
||||
instanceId,
|
||||
scenario: scenario.scenario,
|
||||
machines: [...created, ...routers],
|
||||
networks,
|
||||
pool,
|
||||
};
|
||||
} catch (cause) {
|
||||
throw new RaiseError(instanceId, step, cause);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
/**
|
||||
* Materialise the routers a declaration implies.
|
||||
*
|
||||
* A gateway is the one implicit machine in an otherwise explicit declaration — a scenario
|
||||
* says a segment sits behind one and never names the thing that serves it, because it has
|
||||
* nothing to say about it.
|
||||
*
|
||||
* A router is **scenery, not a node**, so it is a container rather than a virtual machine
|
||||
* (novox/hq ADR 0033). Nothing under test runs on it and no assertion is made about its
|
||||
* internals; it exists so packets behave the way they behave in the world. What it has to
|
||||
* reproduce is kernel behaviour, and a container has the same kernel.
|
||||
*/
|
||||
|
||||
import type { Family, Scenario } from "../declaration/types.ts";
|
||||
import { incus, succeeds } from "../incus/client.ts";
|
||||
import { macFor, networkName } from "./names.ts";
|
||||
import { waitUntilUsable } from "./ready.ts";
|
||||
|
||||
/**
|
||||
* The router image, built once and cached.
|
||||
*
|
||||
* A scenario is a closed address space, so a router has no route to a package repository —
|
||||
* installing nftables at raise time cannot work, and the first attempt failed exactly that
|
||||
* way. So the image is prepared once, with temporary connectivity, and every scenario
|
||||
* afterwards raises from it needing no network at all.
|
||||
*
|
||||
* That is the same property the mesh's own artifacts have: what ships is self-contained,
|
||||
* and a deploy touches no network.
|
||||
*/
|
||||
const ROUTER_IMAGE = "mesh-lab-router";
|
||||
const ROUTER_BASE = "images:alpine/edge";
|
||||
|
||||
/** Wait until the container can actually resolve and fetch — not merely run a command. */
|
||||
async function waitForNetwork(name: string, timeoutSeconds: number): Promise<void> {
|
||||
const deadline = Date.now() + timeoutSeconds * 1000;
|
||||
let lastError = "no attempt made";
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await incus(["exec", name, "--", "apk", "update"], 30_000);
|
||||
return;
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message.split("\n")[0] ?? "" : String(err);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
}
|
||||
throw new Error(
|
||||
`${name} had no working network after ${timeoutSeconds}s — the router image cannot be ` +
|
||||
`built without one. Last error: ${lastError}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the router image if it is missing. One-time, and the only step in the whole lab that
|
||||
* needs the workstation to be online.
|
||||
*/
|
||||
export async function ensureRouterImage(log: (message: string) => void = () => {}): Promise<void> {
|
||||
if (await succeeds(["image", "info", ROUTER_IMAGE], 20_000)) return;
|
||||
|
||||
log(` building the router image (once) — installing nftables into ${ROUTER_BASE}`);
|
||||
const builder = "mlab-router-build";
|
||||
await succeeds(["delete", "--force", builder], 60_000);
|
||||
|
||||
// Default profile on purpose: this is the one container that needs to reach a repository.
|
||||
await incus(["launch", ROUTER_BASE, builder], 300_000);
|
||||
await waitUntilUsable(builder, 120, () => {});
|
||||
|
||||
// `exec` works before the container has an address. Usable means a command runs; it does
|
||||
// not mean the network is up, and the first attempt failed on DNS because those were
|
||||
// treated as the same thing. Wait for the thing actually needed.
|
||||
await waitForNetwork(builder, 60);
|
||||
|
||||
// Not swallowed. A router without nftables is a router that silently does not route, and
|
||||
// an earlier attempt shipped exactly that because the failure was hidden behind `|| true`.
|
||||
await incus(["exec", builder, "--", "apk", "add", "--no-cache", "--update", "nftables"], 180_000);
|
||||
await incus(["exec", builder, "--", "sh", "-c", "command -v nft"], 20_000);
|
||||
|
||||
// The stock image ships `auto eth0 / iface eth0 inet dhcp`, and its boot-time networking
|
||||
// service acts on it — flushing the static address the scenario just set, on eth0 only,
|
||||
// which is why the outside interface came up bare while the inside ones were fine.
|
||||
//
|
||||
// A scenario declares the underlay; a router that reconfigures itself from an image
|
||||
// default is the lab overriding the declaration.
|
||||
await incus([
|
||||
"exec", builder, "--", "sh", "-c",
|
||||
"printf 'auto lo\\niface lo inet loopback\\n' > /etc/network/interfaces",
|
||||
], 30_000);
|
||||
|
||||
await incus(["stop", builder], 120_000);
|
||||
await incus(["publish", builder, "--alias", ROUTER_IMAGE], 300_000);
|
||||
await succeeds(["delete", "--force", builder], 60_000);
|
||||
log(` router image ready`);
|
||||
}
|
||||
|
||||
/** The name a router answers to in `list` and `exec` — scenery, but addressable. */
|
||||
export function routerMachineName(plan: RouterPlan): string {
|
||||
return `gw-${plan.inside.join("-")}`;
|
||||
}
|
||||
|
||||
export interface RouterPlan {
|
||||
/** Router name, one per distinct gateway. */
|
||||
name: string;
|
||||
/** The segment(s) behind this router. Several share one when they share a gateway. */
|
||||
inside: string[];
|
||||
/** The segment this router reaches out through. */
|
||||
outside: string;
|
||||
/** Addresses this router holds on the outside segment — what the world sees. */
|
||||
outsideAddresses: string[];
|
||||
nat: Family[];
|
||||
forwardable: boolean;
|
||||
mappingTtl: string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group segments by the gateway they declare. Identical gateway declarations mean ONE
|
||||
* router, not several — that is what a VLAN-capable router is, and two routers sharing an
|
||||
* external address would not work anyway.
|
||||
*/
|
||||
export function planRouters(scenario: Scenario, instanceId: string): RouterPlan[] {
|
||||
const byGateway = new Map<string, RouterPlan>();
|
||||
|
||||
for (const [segmentName, segment] of Object.entries(scenario.segments)) {
|
||||
const gateway = segment.gateway;
|
||||
if (!gateway) continue;
|
||||
|
||||
const key = `${gateway.to}|${[...gateway.address].sort().join(",")}`;
|
||||
const existing = byGateway.get(key);
|
||||
if (existing) {
|
||||
existing.inside.push(segmentName);
|
||||
continue;
|
||||
}
|
||||
|
||||
byGateway.set(key, {
|
||||
name: `mlab-${instanceId}-gw${byGateway.size}`,
|
||||
inside: [segmentName],
|
||||
outside: gateway.to,
|
||||
outsideAddresses: gateway.address,
|
||||
nat: gateway.nat,
|
||||
forwardable: gateway.forwardable,
|
||||
mappingTtl: gateway.mappingTtl,
|
||||
});
|
||||
}
|
||||
|
||||
return [...byGateway.values()];
|
||||
}
|
||||
|
||||
/** "120s" / "2m" / "90" → seconds. */
|
||||
export function ttlSeconds(text: string | undefined): number | undefined {
|
||||
if (!text) return undefined;
|
||||
const match = /^(\d+)\s*([smh]?)$/.exec(text.trim());
|
||||
if (!match) return undefined;
|
||||
const value = Number(match[1]);
|
||||
return match[2] === "m" ? value * 60 : match[2] === "h" ? value * 3600 : value;
|
||||
}
|
||||
|
||||
function withPrefix(scenario: Scenario, segment: string, address: string): string {
|
||||
const wantV6 = address.includes(":");
|
||||
for (const range of scenario.segments[segment]?.cidr ?? []) {
|
||||
const slash = range.lastIndexOf("/");
|
||||
if (slash === -1) continue;
|
||||
if (range.slice(0, slash).includes(":") === wantV6) return `${address}${range.slice(slash)}`;
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
/** The address a machine holds on a segment, for DNAT targets and as an inside gateway. */
|
||||
function addressOn(scenario: Scenario, machine: string, segment: string, family: Family): string | null {
|
||||
const spec = scenario.machines[machine];
|
||||
if (!spec || spec.at === "detached") return null;
|
||||
for (const attachment of spec.at) {
|
||||
if (attachment.segment !== segment) continue;
|
||||
for (const address of attachment.address) {
|
||||
if ((family === "v6") === address.includes(":")) return address;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The router's own address on an inside segment: the first host address of that range.
|
||||
*
|
||||
* Chosen rather than declared because a scenario has nothing to say about it — the
|
||||
* declaration describes what the world sees the network as, and the inside address is an
|
||||
* implementation detail of the machine serving it.
|
||||
*/
|
||||
function insideAddress(scenario: Scenario, segment: string, family: Family): string | null {
|
||||
for (const range of scenario.segments[segment]?.cidr ?? []) {
|
||||
const slash = range.lastIndexOf("/");
|
||||
if (slash === -1) continue;
|
||||
const base = range.slice(0, slash);
|
||||
const isV6 = base.includes(":");
|
||||
if (isV6 !== (family === "v6")) continue;
|
||||
if (isV6) return `${base.replace(/::$/, "::")}1${range.slice(slash)}`.replace("::1/", "::1/");
|
||||
const octets = base.split(".");
|
||||
octets[3] = "1";
|
||||
return `${octets.join(".")}${range.slice(slash)}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function raiseRouters(
|
||||
scenario: Scenario,
|
||||
instanceId: string,
|
||||
plans: RouterPlan[],
|
||||
log: (message: string) => void = () => {},
|
||||
): Promise<string[]> {
|
||||
const created: string[] = [];
|
||||
|
||||
if (plans.length > 0) await ensureRouterImage(log);
|
||||
|
||||
for (const plan of plans) {
|
||||
if (!(await succeeds(["config", "show", plan.name], 15_000))) {
|
||||
await incus([
|
||||
"init", ROUTER_IMAGE, plan.name,
|
||||
"-c", `user.mesh-lab.instance=${instanceId}`,
|
||||
// Tagged as a machine as well as a router: destroy finds an instance's resources
|
||||
// with one query, and a router that only carried `router=` was left behind — which
|
||||
// then held its networks open, so `destroy` reported removing zero segments.
|
||||
"-c", `user.mesh-lab.machine=${routerMachineName(plan)}`,
|
||||
"-c", `user.mesh-lab.router=${plan.inside.join(",")}`,
|
||||
], 300_000);
|
||||
await succeeds(["config", "device", "remove", plan.name, "eth0"], 15_000);
|
||||
|
||||
// eth0 faces outward, then one interface per segment behind it.
|
||||
const links = [plan.outside, ...plan.inside];
|
||||
for (const [index, segment] of links.entries()) {
|
||||
await incus([
|
||||
"config", "device", "add", plan.name, `eth${index}`, "nic",
|
||||
"nictype=bridged",
|
||||
`parent=${networkName(instanceId, segment)}`,
|
||||
`hwaddr=${macFor(instanceId, `gw-${plan.name}`, index)}`,
|
||||
]);
|
||||
}
|
||||
}
|
||||
await succeeds(["start", plan.name], 60_000);
|
||||
created.push(plan.name);
|
||||
log(` router ${plan.inside.join("+")} → ${plan.outside}`);
|
||||
}
|
||||
|
||||
for (const name of created) {
|
||||
await waitUntilUsable(name, 120, () => {});
|
||||
}
|
||||
|
||||
for (const plan of plans) {
|
||||
await configureRouter(scenario, plan, log);
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
async function sh(name: string, script: string, timeoutMs = 60_000): Promise<void> {
|
||||
await incus(["exec", name, "--", "sh", "-c", script], timeoutMs);
|
||||
}
|
||||
|
||||
async function configureRouter(
|
||||
scenario: Scenario,
|
||||
plan: RouterPlan,
|
||||
log: (message: string) => void,
|
||||
): Promise<void> {
|
||||
// Addresses: eth0 outside, then one per inside segment.
|
||||
const links: { device: string; segment: string; addresses: string[] }[] = [
|
||||
{
|
||||
device: "eth0",
|
||||
segment: plan.outside,
|
||||
addresses: plan.outsideAddresses.map((a) => withPrefix(scenario, plan.outside, a)),
|
||||
},
|
||||
];
|
||||
for (const [index, segment] of plan.inside.entries()) {
|
||||
const addresses: string[] = [];
|
||||
for (const family of ["v4", "v6"] as Family[]) {
|
||||
const address = insideAddress(scenario, segment, family);
|
||||
if (address) addresses.push(address);
|
||||
}
|
||||
links.push({ device: `eth${index + 1}`, segment, addresses });
|
||||
}
|
||||
|
||||
for (const link of links) {
|
||||
// Up first: an address on a down interface is accepted and then not used.
|
||||
await sh(plan.name, `ip link set ${link.device} up`);
|
||||
for (const address of link.addresses) {
|
||||
// `replace` rather than `add`, so re-running is safe and a real failure still fails.
|
||||
await sh(plan.name, `ip addr replace ${address} dev ${link.device}`);
|
||||
}
|
||||
const mtu = scenario.segments[link.segment]?.mtu;
|
||||
if (mtu) await sh(plan.name, `ip link set ${link.device} mtu ${mtu}`);
|
||||
}
|
||||
|
||||
await sh(
|
||||
plan.name,
|
||||
"sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
|
||||
);
|
||||
|
||||
const ttl = ttlSeconds(plan.mappingTtl);
|
||||
if (ttl !== undefined) {
|
||||
// What makes keepalive behaviour testable rather than hoped for: a connection held
|
||||
// through NAT without refreshing dies when the mapping does.
|
||||
//
|
||||
// Read back rather than assumed. These sysctls are not present on every kernel, and a
|
||||
// scenario that declared an expiring mapping and silently got a permanent one would be
|
||||
// the fault this lab exists to catch.
|
||||
await sh(
|
||||
plan.name,
|
||||
`sysctl -w net.netfilter.nf_conntrack_udp_timeout=${ttl} >/dev/null 2>&1; ` +
|
||||
`sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=${ttl} >/dev/null 2>&1; true`,
|
||||
);
|
||||
const readback = await incus(
|
||||
["exec", plan.name, "--", "sh", "-c",
|
||||
"cat /proc/sys/net/netfilter/nf_conntrack_udp_timeout 2>/dev/null || echo missing"],
|
||||
20_000,
|
||||
);
|
||||
if (readback.stdout.trim() !== String(ttl)) {
|
||||
throw new Error(
|
||||
`${plan.name}: mapping_ttl of ${plan.mappingTtl} was declared but conntrack reports ` +
|
||||
`'${readback.stdout.trim()}'. The scenario would silently have permanent mappings.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await applyRules(scenario, plan);
|
||||
log(` ${plan.name}: nat=${plan.nat.join(",") || "none"} forwardable=${plan.forwardable}${ttl ? ` ttl=${ttl}s` : ""}`);
|
||||
}
|
||||
|
||||
/** One ruleset per router, written whole — partial rule edits drift, a whole file does not. */
|
||||
async function applyRules(scenario: Scenario, plan: RouterPlan): Promise<void> {
|
||||
const parts: string[] = ["flush ruleset"];
|
||||
|
||||
for (const family of plan.nat) {
|
||||
const table = family === "v4" ? "ip" : "ip6";
|
||||
parts.push(
|
||||
`table ${table} nat {`,
|
||||
` chain postrouting { type nat hook postrouting priority srcnat; policy accept;`,
|
||||
` oifname "eth0" masquerade`,
|
||||
` }`,
|
||||
` chain prerouting { type nat hook prerouting priority dstnat; policy accept;`,
|
||||
);
|
||||
|
||||
if (plan.forwardable) {
|
||||
for (const [machine, spec] of Object.entries(scenario.machines)) {
|
||||
for (const publication of spec.published ?? []) {
|
||||
if (!plan.inside.includes(publication.on)) continue;
|
||||
const target = addressOn(scenario, machine, publication.on, family);
|
||||
if (!target) continue;
|
||||
const destination = family === "v6" ? `[${target}]` : target;
|
||||
parts.push(
|
||||
` iifname "eth0" tcp dport ${publication.port} dnat to ${destination}:${publication.port}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
parts.push(` }`, `}`);
|
||||
}
|
||||
|
||||
// Filtering: unsolicited inbound, and policy between segments the router serves.
|
||||
const filterLines: string[] = [];
|
||||
if (!plan.forwardable) {
|
||||
// A gateway you do not control: outbound works, nothing initiates inward. That is the
|
||||
// constraint being reproduced, not an implementation limit.
|
||||
filterLines.push(` iifname "eth0" ct state new drop`);
|
||||
}
|
||||
for (const rule of scenario.policy ?? []) {
|
||||
if (rule.allow) continue;
|
||||
const fromIndex = plan.inside.indexOf(rule.from);
|
||||
const toIndex = plan.inside.indexOf(rule.to);
|
||||
if (fromIndex === -1 || toIndex === -1) continue;
|
||||
filterLines.push(` iifname "eth${fromIndex + 1}" oifname "eth${toIndex + 1}" drop`);
|
||||
}
|
||||
|
||||
if (filterLines.length > 0) {
|
||||
parts.push(
|
||||
`table inet filter {`,
|
||||
` chain forward { type filter hook forward priority filter; policy accept;`,
|
||||
` ct state established,related accept`,
|
||||
...filterLines,
|
||||
` }`,
|
||||
`}`,
|
||||
);
|
||||
}
|
||||
|
||||
const ruleset = parts.join("\n");
|
||||
await sh(
|
||||
plan.name,
|
||||
`cat > /tmp/mlab.nft <<'MLABNFT'\n${ruleset}\nMLABNFT\nnft -f /tmp/mlab.nft`,
|
||||
60_000,
|
||||
);
|
||||
}
|
||||
@@ -33,29 +33,6 @@ export class UnsupportedError extends Error {
|
||||
export function assertSupported(scenario: Scenario): void {
|
||||
const missing: string[] = [];
|
||||
|
||||
const withGateways = Object.entries(scenario.segments)
|
||||
.filter(([, segment]) => segment.gateway)
|
||||
.map(([name]) => name);
|
||||
if (withGateways.length > 0) {
|
||||
missing.push(
|
||||
`gateways (segments: ${withGateways.join(", ")}) — no router is materialised, so ` +
|
||||
`nothing routes between segments and NAT does not exist`,
|
||||
);
|
||||
}
|
||||
|
||||
const published = Object.entries(scenario.machines)
|
||||
.filter(([, machine]) => machine.published?.length)
|
||||
.map(([name]) => name);
|
||||
if (published.length > 0) {
|
||||
missing.push(
|
||||
`published ports (machines: ${published.join(", ")}) — requires a gateway to forward through`,
|
||||
);
|
||||
}
|
||||
|
||||
if (scenario.policy?.length) {
|
||||
missing.push("policy between segments — requires a gateway to enforce it");
|
||||
}
|
||||
|
||||
const inbound = Object.entries(scenario.machines)
|
||||
.filter(([, machine]) => machine.inbound === "deny")
|
||||
.map(([name]) => name);
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { parseScenario } from "../src/declaration/parse.ts";
|
||||
import { planRouters, ttlSeconds } from "../src/lifecycle/router.ts";
|
||||
|
||||
test("segments sharing a gateway declaration share ONE router", () => {
|
||||
// That is what a VLAN-capable router is, and two routers sharing an external address
|
||||
// would not work anyway.
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments:
|
||||
pub: { kind: public, cidr: [192.0.2.0/24] }
|
||||
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
iot: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`);
|
||||
const plans = planRouters(scenario, "inst");
|
||||
assert.equal(plans.length, 1, "one gateway declaration, one router");
|
||||
assert.deepEqual(plans[0]?.inside.sort(), ["home", "iot"]);
|
||||
});
|
||||
|
||||
test("different external addresses mean different routers", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments:
|
||||
pub: { kind: public, cidr: [192.0.2.0/24] }
|
||||
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
other: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.6], nat: [v4] } }
|
||||
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`);
|
||||
assert.equal(planRouters(scenario, "inst").length, 2);
|
||||
});
|
||||
|
||||
test("a scenario with no gateways needs no routers", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] } } }`);
|
||||
assert.equal(planRouters(scenario, "inst").length, 0);
|
||||
});
|
||||
|
||||
test("forwardable and nat carry through to the plan", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments:
|
||||
pub: { kind: public, cidr: [192.0.2.0/24] }
|
||||
cafe: { kind: private, cidr: [10.50.0.0/16], gateway: { to: pub, address: [192.0.2.9], nat: [v4], forwardable: false, mapping_ttl: 30s } }
|
||||
machines: { a: { at: { segment: cafe, address: [10.50.0.9] } } }`);
|
||||
const plan = planRouters(scenario, "inst")[0];
|
||||
assert.equal(plan?.forwardable, false);
|
||||
assert.deepEqual(plan?.nat, ["v4"]);
|
||||
assert.equal(plan?.mappingTtl, "30s");
|
||||
});
|
||||
|
||||
test("mapping ttl parses the forms a declaration uses", () => {
|
||||
assert.equal(ttlSeconds("30s"), 30);
|
||||
assert.equal(ttlSeconds("120s"), 120);
|
||||
assert.equal(ttlSeconds("2m"), 120);
|
||||
assert.equal(ttlSeconds("1h"), 3600);
|
||||
assert.equal(ttlSeconds("90"), 90);
|
||||
assert.equal(ttlSeconds(undefined), undefined);
|
||||
assert.equal(ttlSeconds("soon"), undefined);
|
||||
});
|
||||
+34
-19
@@ -5,8 +5,9 @@ import { assertSupported, UnsupportedError } from "../src/lifecycle/supported.ts
|
||||
|
||||
/**
|
||||
* A declaration the runtime silently ignores is the fault this lab exists to catch —
|
||||
* novox/hq 04-ISSUES/003, where a firewall key is declared in five manifests and read by
|
||||
* no code. These tests exist so the lab never commits it.
|
||||
* novox/hq 04-ISSUES/003, where a firewall key is declared in five manifests and read by no
|
||||
* code. These tests exist so the lab never commits it, and they move as the runtime catches
|
||||
* up with the model.
|
||||
*/
|
||||
|
||||
const withGateway = `scenario: x
|
||||
@@ -15,49 +16,63 @@ segments:
|
||||
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`;
|
||||
|
||||
test("a scenario with no unimplemented features is raisable", () => {
|
||||
test("a plain scenario is raisable", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] } } }`);
|
||||
assert.doesNotThrow(() => assertSupported(scenario));
|
||||
});
|
||||
|
||||
test("a declared gateway is refused rather than silently absent", () => {
|
||||
assert.throws(() => assertSupported(parseScenario(withGateway)), UnsupportedError);
|
||||
test("gateways are implemented — a router is materialised for them", () => {
|
||||
assert.doesNotThrow(() => assertSupported(parseScenario(withGateway)));
|
||||
});
|
||||
|
||||
test("the refusal names every missing capability, not just the first", () => {
|
||||
test("published ports and policy are implemented", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments:
|
||||
pub: { kind: public, cidr: [192.0.2.0/24] }
|
||||
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
policy: [{ from: home, to: pub, allow: false }]
|
||||
iot: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
|
||||
policy: [{ from: iot, to: home, allow: false }]
|
||||
machines:
|
||||
a:
|
||||
at: { segment: home, address: [192.168.1.9] }
|
||||
published: [{ port: 443, on: home }]
|
||||
inbound: deny
|
||||
published: [{ port: 443, on: home }]`);
|
||||
assert.doesNotThrow(() => assertSupported(scenario));
|
||||
});
|
||||
|
||||
test("inbound: deny is still refused rather than silently absent", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } }`);
|
||||
assert.throws(() => assertSupported(scenario), UnsupportedError);
|
||||
});
|
||||
|
||||
test("place is still refused — there is nothing to place yet", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] } } }
|
||||
place: { all: [host] }`);
|
||||
assert.throws(() => assertSupported(scenario), UnsupportedError);
|
||||
});
|
||||
|
||||
test("the refusal names every gap, not just the first", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } }
|
||||
place: { all: [host] }`);
|
||||
try {
|
||||
assertSupported(scenario);
|
||||
assert.fail("should have refused");
|
||||
} catch (err) {
|
||||
const missing = (err as UnsupportedError).missing;
|
||||
assert.ok(missing.length >= 5, `expected every gap named, got ${missing.length}`);
|
||||
assert.equal((err as UnsupportedError).missing.length, 2);
|
||||
assert.match(err instanceof Error ? err.message : "", /silently lacks them/);
|
||||
}
|
||||
});
|
||||
|
||||
test("inbound: allow is not a missing capability — only deny needs enforcing", () => {
|
||||
test("inbound: allow is not a gap — only deny needs enforcing", () => {
|
||||
const scenario = parseScenario(`scenario: x
|
||||
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
|
||||
machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: allow } }`);
|
||||
assert.doesNotThrow(() => assertSupported(scenario));
|
||||
});
|
||||
|
||||
test("validation and raisability are different questions", () => {
|
||||
// The declaration model is complete; the runtime is not. A scenario may be valid and
|
||||
// still not raisable, and conflating the two would hide the gap.
|
||||
assert.doesNotThrow(() => parseScenario(withGateway), "should validate");
|
||||
assert.throws(() => assertSupported(parseScenario(withGateway)), "should not raise");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user