2026-08-25 23:02:29 +00:00
9 changed files with 295 additions and 31 deletions
@@ -110,10 +110,11 @@ than ignored:
 | gateways, NAT, masquerade | **works** |
 | `published:` ports (DNAT through the gateway's address) | **works** |
 | `mapping_ttl:` (conntrack timeout) | **works**, and verified after setting — a declared expiry that silently did not apply would be the fault this catches |
-| `forwardable: false` | implemented, **not yet verified by running** |
-| `policy:` between segments | implemented, **not yet verified by running** |
-| `inbound: deny` | **refused at raise** |
-| `place:` | **refused at raise** |
+| `forwardable: false` | **works** — outbound only, no DNAT, unsolicited inbound dropped |
+| `policy:` between segments | **works**, asymmetric |
+| `inbound: deny` | **works** — host firewall, read back after applying |
+| several public networks, routed not bridged | **works** — a transit router, never a shared bridge |
+| `place:` | **refused at raise** — the node host it would place does not exist yet |

 `raise` refuses a scenario declaring anything in the lower half, naming every gap. It does not
 raise a mesh that silently lacks what it declared — that is the fault this lab exists to catch
@@ -138,12 +139,29 @@ something under test (`novox/hq` ADR 0033).
 gateway, reached from a machine on a routable address:

 ```
-home-server -> anchor                       0% loss, through masquerade
-anchor -> 192.168.1.135  (private, direct)  unreachable ✓
-anchor -> 192.0.2.50:8080 (the GATEWAY)     HTTP 200
+home-server -> anchor                        0% loss, through masquerade
+anchor -> 192.168.1.135  (private, direct)   unreachable ✓
+anchor -> 192.0.2.50:8080 (the GATEWAY)      HTTP 200
+home -> devices  (policy allow)              reachable ✓
+devices -> home  (policy deny)               blocked ✓
+roamer behind unforwardable NAT -> anchor    reachable ✓  (outbound only)
+anchor -> roamer                             unreachable ✓
+workstation with inbound: deny, dialling out reachable ✓  (defended, not disconnected)
+home-server -> workstation                   refused ✓
 ```

-The last line is the case research 004 says only exists in production.
+The third line is the case research 004 says only exists in production.
+
+**Routed, never bridged**, proven rather than asserted — ping TTL across the full topology:
+
+```
+within one segment                           ttl=64   no hops
+across two unrelated public networks         ttl=62   gateway + transit
+multicast between public networks            0 replies
+```
+
+A flat "internet" would have shown ttl=64 and answered multicast, which would have let a node
+discover a peer it could never reach in production — and report success.

 Machines boot concurrently, so a second machine costs seconds rather than doubling the wait.
 Nearly all of the remaining time is boot, which cannot be avoided.
@@ -0,0 +1,65 @@
+# Two things production has and a flat lab cannot show.
+#
+# `devices` and `home` sit behind ONE router — identical gateway declarations — with a
+# policy allowing home→devices and denying the reverse. That is an ordinary segmented
+# household router, and the asymmetry is the normal case.
+#
+# `cafe` sits behind a gateway we do not control. Outbound works; nothing initiates
+# inward, and nothing can be published there at all.
+scenario: segmented-and-unforwardable
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+  home:
+    kind: private
+    cidr: [192.168.1.0/24]
+    gateway:
+      to: hosting
+      address: [192.0.2.50]
+      nat: [v4]
+      forwardable: true
+      mapping_ttl: 120s
+
+  devices:
+    kind: private
+    cidr: [192.168.30.0/24]
+    gateway:
+      to: hosting
+      address: [192.0.2.50]     # identical → the SAME router
+      nat: [v4]
+      forwardable: true
+      mapping_ttl: 120s
+
+  cafe:
+    kind: private
+    cidr: [10.50.0.0/16]
+    gateway:
+      to: hosting
+      address: [192.0.2.80]
+      nat: [v4]
+      forwardable: false        # carrier-grade NAT, or simply not ours
+      mapping_ttl: 30s
+
+policy:
+  - { from: devices, to: home, allow: false }
+  - { from: home, to: devices, allow: true }
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    inbound: allow
+
+  home-server:
+    at: { segment: home, address: [192.168.1.135] }
+    inbound: allow
+
+  thermostat:
+    at: { segment: devices, address: [192.168.30.20] }
+    inbound: allow
+
+  roamer:
+    at: { segment: cafe, address: [10.50.3.23] }
+    inbound: allow
@@ -74,8 +74,8 @@ machines:
    at: { segment: home, address: [192.168.1.98, "2001:db8:b:1::98"] }
    inbound: deny

-place:
-  all: [host]
-  anchor: [substrate]
+# No `place:` yet. The node host it would place does not exist — this lab is being built to
+# develop it, and the lab refuses declarations it cannot materialise rather than raising a
+# mesh that silently lacks them.

 snapshot: raised
@@ -13,9 +13,10 @@
 * See novox/hq 02-DECISIONS/0031-the-lab-provides-the-underlay.md
 */

-import type { Scenario } from "../declaration/types.ts";
+import type { Attachment, Scenario } from "../declaration/types.ts";
 import { incus } from "../incus/client.ts";
 import { macFor } from "./names.ts";
+import { transitAddress } from "./router.ts";

 export interface Wire {
  device: string;
@@ -120,10 +121,14 @@ export async function applyDefaultRoutes(
    const name = machineNames.get(machine);
    if (!name) continue;

-    // A machine behind a gateway routes through it. A machine attached directly to a public
-    // segment has nowhere to default to, and should not pretend otherwise.
+    // A machine behind a gateway routes through it. A machine sitting directly on a public
+    // segment routes through transit instead — otherwise it can reach its own network and
+    // nothing else, which is not what being on the internet means.
    const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway);
-    if (!behind) continue;
+    if (!behind) {
+      await routeViaTransit(scenario, spec, name);
+      continue;
+    }

    const index = spec.at.indexOf(behind);
    for (const range of scenario.segments[behind.segment]?.cidr ?? []) {
@@ -143,3 +148,27 @@ export async function applyDefaultRoutes(
    log(`  routed ${machine} via its gateway on ${behind.segment}`);
  }
 }
+
+/** A machine on a public segment reaches the other public networks through transit. */
+async function routeViaTransit(
+  scenario: Scenario,
+  spec: { at: Attachment[] | "detached" },
+  name: string,
+): Promise<void> {
+  if (spec.at === "detached") return;
+  const onPublic = spec.at.find((a) => scenario.segments[a.segment]?.kind === "public");
+  if (!onPublic) return;
+
+  const index = spec.at.indexOf(onPublic);
+  for (const cidr of scenario.segments[onPublic.segment]?.cidr ?? []) {
+    const via = transitAddress(cidr);
+    if (!via) continue;
+    const gateway = via.slice(0, via.lastIndexOf("/"));
+    const family = gateway.includes(":") ? "-6" : "-4";
+    await incus(
+      ["exec", name, "--", "sh", "-c",
+       `ip ${family} route replace default via ${gateway} dev $(ip -o link | awk -F': ' 'NR==${index + 2}{print $2}') 2>/dev/null || true`],
+      30_000,
+    );
+  }
+}
@@ -0,0 +1,59 @@
+/**
+ * `inbound: deny` — a host firewall on the machine itself.
+ *
+ * Distinct from NAT and behaving differently: a machine can be perfectly routable and
+ * still refuse everything unsolicited, which is the normal state of a v6-addressed
+ * machine. Without this, v6 addressing would silently imply reachability, and a scenario
+ * that said a machine refuses traffic would produce one that accepts it.
+ *
+ * Established and related traffic is accepted, so the machine can still dial out. That is
+ * what a host firewall does; a machine that could not reach anything would be reproducing
+ * a disconnected machine rather than a defended one.
+ */
+
+import type { Scenario } from "../declaration/types.ts";
+import { incus } from "../incus/client.ts";
+
+const RULESET = `flush ruleset
+table inet mlab {
+  chain input {
+    type filter hook input priority filter; policy drop;
+    ct state established,related accept
+    iif lo accept
+    ct state invalid drop
+  }
+}
+`;
+
+export async function applyHostFirewalls(
+  scenario: Scenario,
+  machineNames: Map<string, string>,
+  log: (message: string) => void = () => {},
+): Promise<void> {
+  for (const [machine, spec] of Object.entries(scenario.machines)) {
+    if (spec.inbound !== "deny") continue;
+    const name = machineNames.get(machine);
+    if (!name) continue;
+
+    await incus(
+      ["exec", name, "--", "sh", "-c",
+       `cat > /tmp/mlab-host.nft <<'MLABNFT'\n${RULESET}MLABNFT\nnft -f /tmp/mlab-host.nft`],
+      60_000,
+    );
+
+    // Read back. A declared refusal that silently did not apply is the fault this lab
+    // exists to catch, and a ruleset that failed to load leaves the machine wide open —
+    // which looks exactly like a machine that is working.
+    const check = await incus(
+      ["exec", name, "--", "sh", "-c", "nft list table inet mlab >/dev/null 2>&1 && echo present || echo absent"],
+      20_000,
+    );
+    if (check.stdout.trim() !== "present") {
+      throw new Error(
+        `${machine}: inbound: deny was declared but the ruleset is not loaded — the machine ` +
+          `would accept traffic the scenario says it refuses`,
+      );
+    }
+    log(`  ${machine} refuses unsolicited inbound`);
+  }
+}
@@ -20,7 +20,8 @@ import { machineName, macFor, networkName, newInstanceId } from "./names.ts";
 import { waitUntilAllUsable } from "./ready.ts";
 import { applyAddresses, applyDefaultRoutes } from "./address.ts";
 import { assertSupported } from "./supported.ts";
-import { planRouters, raiseRouters } from "./router.ts";
+import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
+import { applyHostFirewalls } from "./firewall.ts";

 /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
 const COW_DRIVERS = ["btrfs", "zfs"];
@@ -196,12 +197,22 @@ export async function raise(
    step = "applying declared addresses";
    await applyAddresses(scenario, instanceId, byMachine, log);

+    // Transit first: a gateway's default route points at it, so it has to exist.
+    step = "wiring the public networks together";
+    const transit = await raiseTransit(scenario, instanceId, log);
+
    step = "raising routers";
    const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
+    if (transit) routers.push(transit);

    step = "routing machines through their gateways";
    await applyDefaultRoutes(scenario, byMachine, log);

+    // Last: a machine that refuses inbound must still have been reachable while the lab
+    // was configuring it.
+    step = "applying host firewalls";
+    await applyHostFirewalls(scenario, byMachine, log);
+
    return {
      instanceId,
      scenario: scenario.scenario,
@@ -91,6 +91,23 @@ export async function ensureRouterImage(log: (message: string) => void = () => {
  log(`  router image ready`);
 }

+/**
+ * The address the transit router holds on a public segment: the last usable host address.
+ *
+ * Chosen rather than declared, like a gateway's inside address — a scenario has nothing to
+ * say about the internet's own routers, only about the networks they connect.
+ */
+export function transitAddress(cidr: string): string | null {
+  const slash = cidr.lastIndexOf("/");
+  if (slash === -1) return null;
+  const base = cidr.slice(0, slash);
+  const prefix = cidr.slice(slash);
+  if (base.includes(":")) return `${base.replace(/::$/, "")}::fffe${prefix}`;
+  const octets = base.split(".");
+  octets[3] = "254";
+  return `${octets.join(".")}${prefix}`;
+}
+
 /** The name a router answers to in `list` and `exec` — scenery, but addressable. */
 export function routerMachineName(plan: RouterPlan): string {
  return `gw-${plan.inside.join("-")}`;
@@ -197,6 +214,70 @@ function insideAddress(scenario: Scenario, segment: string, family: Family): str
  return null;
 }

+/**
+ * Wire the public segments together.
+ *
+ * The internet is not a network — it is unrelated networks that route to each other, many
+ * hops apart with no shared broadcast domain. So public segments are separate links joined
+ * by a router, never bridged: bridging them would make ARP adjacency, non-decrementing TTL
+ * and crossing multicast true in the lab and false in production, and the mesh has already
+ * been bitten by multicast name resolution.
+ *
+ * One transit router, an interface on every public segment, forwarding and no translation.
+ * It is the closest thing the lab has to "the internet", and it is deliberately dumb.
+ */
+export async function raiseTransit(
+  scenario: Scenario,
+  instanceId: string,
+  log: (message: string) => void = () => {},
+): Promise<string | null> {
+  const publicSegments = Object.entries(scenario.segments)
+    .filter(([, segment]) => segment.kind === "public")
+    .map(([name]) => name);
+
+  // One public network needs no transit: everything on it is already adjacent.
+  if (publicSegments.length < 2) return null;
+
+  const name = `mlab-${instanceId}-transit`;
+  if (!(await succeeds(["config", "show", name], 15_000))) {
+    await incus([
+      "init", ROUTER_IMAGE, name,
+      "-c", `user.mesh-lab.instance=${instanceId}`,
+      "-c", "user.mesh-lab.machine=transit",
+      "-c", `user.mesh-lab.transit=${publicSegments.join(",")}`,
+    ], 300_000);
+    await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
+    for (const [index, segment] of publicSegments.entries()) {
+      await incus([
+        "config", "device", "add", name, `eth${index}`, "nic",
+        "nictype=bridged",
+        `parent=${networkName(instanceId, segment)}`,
+        `hwaddr=${macFor(instanceId, "transit", index)}`,
+      ]);
+    }
+  }
+  await succeeds(["start", name], 60_000);
+  await waitUntilUsable(name, 120, () => {});
+
+  for (const [index, segment] of publicSegments.entries()) {
+    const device = `eth${index}`;
+    await sh(name, `ip link set ${device} up`);
+    for (const cidr of scenario.segments[segment]?.cidr ?? []) {
+      const address = transitAddress(cidr);
+      if (address) await sh(name, `ip addr replace ${address} dev ${device}`);
+    }
+    const mtu = scenario.segments[segment]?.mtu;
+    if (mtu) await sh(name, `ip link set ${device} mtu ${mtu}`);
+  }
+
+  await sh(
+    name,
+    "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
+  );
+  log(`  transit router across ${publicSegments.join(", ")}`);
+  return name;
+}
+
 export async function raiseRouters(
  scenario: Scenario,
  instanceId: string,
@@ -289,6 +370,17 @@ async function configureRouter(
    "sysctl -w net.ipv4.ip_forward=1 >/dev/null; sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null",
  );

+  // A gateway reaches other public networks the way anything does: through transit. Without
+  // this it can only reach its own outside segment, and every scenario with more than one
+  // public network becomes a set of islands.
+  for (const cidr of scenario.segments[plan.outside]?.cidr ?? []) {
+    const via = transitAddress(cidr);
+    if (!via) continue;
+    const gateway = via.slice(0, via.lastIndexOf("/"));
+    const family = gateway.includes(":") ? "-6" : "-4";
+    await sh(plan.name, `ip ${family} route replace default via ${gateway} dev eth0 2>/dev/null || true`);
+  }
+
  const ttl = ttlSeconds(plan.mappingTtl);
  if (ttl !== undefined) {
    // What makes keepalive behaviour testable rather than hoped for: a connection held
@@ -33,16 +33,6 @@ export class UnsupportedError extends Error {
 export function assertSupported(scenario: Scenario): void {
  const missing: string[] = [];

-  const inbound = Object.entries(scenario.machines)
-    .filter(([, machine]) => machine.inbound === "deny")
-    .map(([name]) => name);
-  if (inbound.length > 0) {
-    missing.push(
-      `inbound: deny (machines: ${inbound.join(", ")}) — no host firewall is configured, so ` +
-        `these machines would accept traffic the scenario says they refuse`,
-    );
-  }
-
  if (scenario.place && Object.keys(scenario.place).length > 0) {
    missing.push(
      "place — nothing is placed inside the machines yet; they are raised bare",
@@ -41,11 +41,11 @@ machines:
  assert.doesNotThrow(() => assertSupported(scenario));
 });

-test("inbound: deny is still refused rather than silently absent", () => {
+test("inbound: deny is implemented — a host firewall is applied and read back", () => {
  const scenario = parseScenario(`scenario: x
 segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
 machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } }`);
-  assert.throws(() => assertSupported(scenario), UnsupportedError);
+  assert.doesNotThrow(() => assertSupported(scenario));
 });

 test("place is still refused — there is nothing to place yet", () => {
@@ -56,16 +56,16 @@ place: { all: [host] }`);
  assert.throws(() => assertSupported(scenario), UnsupportedError);
 });

-test("the refusal names every gap, not just the first", () => {
+test("the refusal explains what would silently be missing", () => {
  const scenario = parseScenario(`scenario: x
 segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
-machines: { a: { at: { segment: net, address: [192.0.2.1] }, inbound: deny } }
+machines: { a: { at: { segment: net, address: [192.0.2.1] } } }
 place: { all: [host] }`);
  try {
    assertSupported(scenario);
    assert.fail("should have refused");
  } catch (err) {
-    assert.equal((err as UnsupportedError).missing.length, 2);
+    assert.equal((err as UnsupportedError).missing.length, 1);
    assert.match(err instanceof Error ? err.message : "", /silently lacks them/);
  }
 });