A machine says which networks it routes, and its filter forwards them

The derived filter denies forwarding by default and then allows the container
runtime's two default pools, named in this code with a comment saying a machine
configured otherwise needs to say so -- and no way to say it. So the filter was
right on a machine using the defaults and silently wrong on any other.

Measured today: flipping a workstation to the derived filter cut egress for five
of its container networks and for every network its test beds create, because
those come from ranges the defaults do not cover. Nothing reported a fault; the
guests just could not reach anything, while the machine reported it had applied
what it was told.

A node-level fact beside the public domain, because the machine routes them and
the module that loads the filter may be replaced. Added to the defaults, never
replacing them. Their guests also keep address and name service, without which a
network does not work at all, and the converge preview now says what a machine
routes instead of leaving it to a sentence about what it cannot preview.

cmd/mesh-controller/adoption.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"net"
 	"slices"
 	"sort"
 	"strings"
@@ -309,7 +310,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
 		return "", err
 	}
 	derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
-		outward: plan.PublicDomain != ""}
+		outward: plan.PublicDomain != "", routed: with.Routed}
 	preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
 	preview += "\n\n  preview " + saw
 	if !yes {
@@ -414,6 +415,17 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
 	b.WriteString("    not previewed: traffic the machine routes that is not a published port " +
 		"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
 		"declares it\n")
+	// What it routes, said rather than left to the sentence above (novox/hq ADR 0137). The filter
+	// forwards the container runtime's own default pools without being told; anything else is this
+	// list, and a machine whose guests live outside those pools and names none of them loses their
+	// egress at the flip, silently, which is how this was found.
+	if len(derived.routed) > 0 {
+		b.WriteString(fmt.Sprintf("    it routes: %s — kept forwarded, and their guests keep "+
+			"address and name service\n", strings.Join(derived.routed, ", ")))
+	} else {
+		b.WriteString("    it routes: nothing said, so only the container runtime's own default " +
+			"pools are forwarded — `node networks <node> <cidr>...` if its guests live elsewhere\n")
+	}
 
 	isTaken := map[string]bool{}
 	for _, m := range taken {
@@ -473,6 +485,9 @@ type derivedFilter struct {
 	// mesh is every address on the private network; outward says the machine faces outside.
 	mesh    []string
 	outward bool
+	// routed is the networks this machine says it routes for what it hosts (novox/hq ADR 0137):
+	// their guests keep address and name service, and what they send onward keeps being forwarded.
+	routed []string
 }
 
 // closesOutside is what a narrowing from everywhere to the private network is called: it closes.
@@ -498,6 +513,13 @@ func (d derivedFilter) fate(r inventory.Reach) string {
 			return "stays open — the mesh's own, from anywhere"
 		}
 	}
+	// A guest on a network this machine routes asks it for an address and for names, and those two
+	// arrive here (novox/hq ADR 0137). Matched on the listener's own address: a resolver bound to a
+	// bridge in one of those networks is the one its guests ask.
+	if within(r.Address, d.routed) &&
+		((r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53)) {
+		return "stays open — address and name service for a network this machine routes"
+	}
 	for _, rule := range d.rules {
 		if rule.Port != r.Port || rule.Protocol != r.Protocol {
 			continue
@@ -520,6 +542,24 @@ func (d derivedFilter) fate(r inventory.Reach) string {
 	return "WILL CLOSE — no module assigned here declares it"
 }
 
+// within says whether an address the machine reported sits inside one of the networks it routes.
+func within(address string, networks []string) bool {
+	ip := net.ParseIP(strings.Trim(address, "[]"))
+	if ip == nil {
+		return false
+	}
+	for _, n := range networks {
+		_, block, err := net.ParseCIDR(n)
+		if err != nil {
+			continue
+		}
+		if block.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
 // countReachable is how much of a node's account of itself names something off the machine.
 // Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
 // so a report of loopback alone says nothing about what the filter would close.

cmd/mesh-controller/main.go

@@ -148,6 +148,9 @@ func usage() {
   node public-domain <name>      the domain it composes its routed names under
   node public-domain <name> <d>  ...set it to d
   node public-domain <name> --clear   ...it faces the outside no longer
+  node networks <name>           the networks it routes for what it hosts
+  node networks <name> <cidr>... ...set them; its filter forwards these too
+  node networks <name> --clear   ...only the container runtime's own
   token issue --node <name>      a one-time right to join, for an existing record
   token issue --new <name>       create the record and issue for it
   token issue ... --adopted      ...for a machine in use, which joins adopted

cmd/mesh-controller/nodes.go

@@ -21,7 +21,8 @@ import (
 
 func nodeCommand(ctx context.Context, args []string) error {
 	if len(args) == 0 {
-		return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
+		return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage +
+			", or " + networksUsage)
 	}
 	open, err := openStores(ctx)
 	if err != nil {
@@ -66,6 +67,12 @@ func nodeCommand(ctx context.Context, args []string) error {
 		// because the damage is already done by the time it prints.
 		return publicDomain(ctx, inv, args[1:])
 
+	case "networks":
+		// The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived
+		// filter must keep forwarding, beyond the container runtime's own default pools which it
+		// allows without being told. Reports with no argument, for the same reason the domain does.
+		return nodeNetworks(ctx, inv, args[1:])
+
 	case "account":
 		// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
 		// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
@@ -144,6 +151,67 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st
 	return nil
 }
 
+const networksUsage = "node networks <name> — what it routes now; " +
+	"<name> <cidr>... to set them; <name> --clear to route only the container runtime's own"
+
+// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts.
+//
+// The same three forms as the domain above, and the read-shaped one reports rather than clearing,
+// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it
+// by asking a question is not a mistake anybody can see afterwards.
+func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error {
+	set := flag.NewFlagSet("node networks", flag.ContinueOnError)
+	clear := set.Bool("clear", false,
+		"route only the container runtime's own default pools, as a machine that has said nothing does")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) == 0 {
+		return errors.New(networksUsage)
+	}
+	node := positionals[0]
+
+	switch {
+	case *clear && len(positionals) > 1:
+		return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+
+			"things and the mesh will not choose between them", node, strings.Join(positionals[1:], " "))
+
+	case *clear:
+		if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil {
+			return err
+		}
+		fmt.Printf("%s routes only the container runtime's own default pools\n", node)
+		fmt.Printf("  run `push %s` to send its filter\n", node)
+		return nil
+
+	case len(positionals) > 1:
+		if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil {
+			return err
+		}
+		fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", "))
+		fmt.Printf("  its filter forwards them, and their guests keep address and name service\n")
+		fmt.Printf("  run `push %s` to send it\n", node)
+		return nil
+
+	default:
+		if _, err := inv.NodeByName(ctx, node); err != nil {
+			return err
+		}
+		networks, err := inv.RoutedNetworksOf(ctx, node)
+		if err != nil {
+			return err
+		}
+		if len(networks) == 0 {
+			fmt.Printf("%s routes only the container runtime's own default pools\n", node)
+			fmt.Printf("  `node networks %s <cidr>...` if its guests live elsewhere\n", node)
+			return nil
+		}
+		fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", "))
+		return nil
+	}
+}
+
 const publicDomainUsage = "node public-domain <name> — what it is now; " +
 	"<name> <domain> to set it; <name> --clear to take it away"
 

cmd/mesh-controller/plan.go

@@ -640,13 +640,19 @@ func renderingFor(ctx context.Context, open *stores, node string,
 	if err != nil {
 		return catalogue.Rendering{}, inventory.Node{}, err
 	}
+	// The networks this machine routes for what it hosts, which the derived filter must forward for
+	// (novox/hq ADR 0137).
+	routed, err := inv.RoutedNetworksOf(ctx, node)
+	if err != nil {
+		return catalogue.Rendering{}, inventory.Node{}, err
+	}
 	return catalogue.Rendering{
 		BusMembership: memberships[node],
 		Settings:      settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
 		Machines: machines,
 		Suffix:   overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
-		Kept: kept, Adopted: record.Adopted,
+		Kept: kept, Adopted: record.Adopted, Routed: routed,
 		Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
 		BusUsers: busUsers,
 	}, record, nil