diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 6b9d95b..2e5bdc0 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "net" "slices" "sort" "strings" @@ -309,7 +310,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s return "", err } derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, - outward: plan.PublicDomain != ""} + outward: plan.PublicDomain != "", routed: with.Routed} preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter]) preview += "\n\n preview " + saw if !yes { @@ -414,6 +415,17 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, b.WriteString(" not previewed: traffic the machine routes that is not a published port " + "(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " + "declares it\n") + // What it routes, said rather than left to the sentence above (novox/hq ADR 0137). The filter + // forwards the container runtime's own default pools without being told; anything else is this + // list, and a machine whose guests live outside those pools and names none of them loses their + // egress at the flip, silently, which is how this was found. + if len(derived.routed) > 0 { + b.WriteString(fmt.Sprintf(" it routes: %s — kept forwarded, and their guests keep "+ + "address and name service\n", strings.Join(derived.routed, ", "))) + } else { + b.WriteString(" it routes: nothing said, so only the container runtime's own default " + + "pools are forwarded — `node networks ...` if its guests live elsewhere\n") + } isTaken := map[string]bool{} for _, m := range taken { @@ -473,6 +485,9 @@ type derivedFilter struct { // mesh is every address on the private network; outward says the machine faces outside. mesh []string outward bool + // routed is the networks this machine says it routes for what it hosts (novox/hq ADR 0137): + // their guests keep address and name service, and what they send onward keeps being forwarded. + routed []string } // closesOutside is what a narrowing from everywhere to the private network is called: it closes. @@ -498,6 +513,13 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "stays open — the mesh's own, from anywhere" } } + // A guest on a network this machine routes asks it for an address and for names, and those two + // arrive here (novox/hq ADR 0137). Matched on the listener's own address: a resolver bound to a + // bridge in one of those networks is the one its guests ask. + if within(r.Address, d.routed) && + ((r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53)) { + return "stays open — address and name service for a network this machine routes" + } for _, rule := range d.rules { if rule.Port != r.Port || rule.Protocol != r.Protocol { continue @@ -520,6 +542,24 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "WILL CLOSE — no module assigned here declares it" } +// within says whether an address the machine reported sits inside one of the networks it routes. +func within(address string, networks []string) bool { + ip := net.ParseIP(strings.Trim(address, "[]")) + if ip == nil { + return false + } + for _, n := range networks { + _, block, err := net.ParseCIDR(n) + if err != nil { + continue + } + if block.Contains(ip) { + return true + } + } + return false +} + // countReachable is how much of a node's account of itself names something off the machine. // Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it, // so a report of loopback alone says nothing about what the filter would close. diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 28d52a1..6b0f312 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -148,6 +148,9 @@ func usage() { node public-domain the domain it composes its routed names under node public-domain ...set it to d node public-domain --clear ...it faces the outside no longer + node networks the networks it routes for what it hosts + node networks ... ...set them; its filter forwards these too + node networks --clear ...only the container runtime's own token issue --node a one-time right to join, for an existing record token issue --new create the record and issue for it token issue ... --adopted ...for a machine in use, which joins adopted diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 6f54179..59aa7b8 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -21,7 +21,8 @@ import ( func nodeCommand(ctx context.Context, args []string) error { if len(args) == 0 { - return errors.New("node add , node list, node show , or " + publicDomainUsage) + return errors.New("node add , node list, node show , " + publicDomainUsage + + ", or " + networksUsage) } open, err := openStores(ctx) if err != nil { @@ -66,6 +67,12 @@ func nodeCommand(ctx context.Context, args []string) error { // because the damage is already done by the time it prints. return publicDomain(ctx, inv, args[1:]) + case "networks": + // The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived + // filter must keep forwarding, beyond the container runtime's own default pools which it + // allows without being told. Reports with no argument, for the same reason the domain does. + return nodeNetworks(ctx, inv, args[1:]) + case "account": // The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is // owned by and which account `ssh ` uses. Reports with no argument; sets with one; @@ -144,6 +151,67 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st return nil } +const networksUsage = "node networks — what it routes now; " + + " ... to set them; --clear to route only the container runtime's own" + +// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts. +// +// The same three forms as the domain above, and the read-shaped one reports rather than clearing, +// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it +// by asking a question is not a mistake anybody can see afterwards. +func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error { + set := flag.NewFlagSet("node networks", flag.ContinueOnError) + clear := set.Bool("clear", false, + "route only the container runtime's own default pools, as a machine that has said nothing does") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) == 0 { + return errors.New(networksUsage) + } + node := positionals[0] + + switch { + case *clear && len(positionals) > 1: + return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+ + "things and the mesh will not choose between them", node, strings.Join(positionals[1:], " ")) + + case *clear: + if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil { + return err + } + fmt.Printf("%s routes only the container runtime's own default pools\n", node) + fmt.Printf(" run `push %s` to send its filter\n", node) + return nil + + case len(positionals) > 1: + if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil { + return err + } + fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", ")) + fmt.Printf(" its filter forwards them, and their guests keep address and name service\n") + fmt.Printf(" run `push %s` to send it\n", node) + return nil + + default: + if _, err := inv.NodeByName(ctx, node); err != nil { + return err + } + networks, err := inv.RoutedNetworksOf(ctx, node) + if err != nil { + return err + } + if len(networks) == 0 { + fmt.Printf("%s routes only the container runtime's own default pools\n", node) + fmt.Printf(" `node networks %s ...` if its guests live elsewhere\n", node) + return nil + } + fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", ")) + return nil + } +} + const publicDomainUsage = "node public-domain — what it is now; " + " to set it; --clear to take it away" diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 0280117..a70a29f 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -640,13 +640,19 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + // The networks this machine routes for what it hosts, which the derived filter must forward for + // (novox/hq ADR 0137). + routed, err := inv.RoutedNetworksOf(ctx, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, - Kept: kept, Adopted: record.Adopted, + Kept: kept, Adopted: record.Adopted, Routed: routed, Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, BusUsers: busUsers, }, record, nil diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index a592599..6a44726 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -124,6 +124,11 @@ type Rendering struct { // nothing on this node keeps them, or the mesh has no operator key. Kept *KeptExport + // Routed is the networks this machine routes for what it hosts, beyond the container runtime's + // own default pools, which the filter allows without being told (novox/hq ADR 0137). A node-level + // fact: the machine routes them, and the module that loads the filter may be replaced. + Routed []string + // Foundation is the ports the mesh itself needs reachable on every machine, which no module // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker // is the one that matters: a machine dials it to enrol, and a firewall derived only from @@ -345,7 +350,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri if err != nil { return nil, err } - filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation) + filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation, with.Routed) var out []map[string]any for _, m := range r.Modules { diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index fd3c78f..bd61b40 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -230,7 +230,7 @@ const SSHPort = 22 // It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can // repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing // any module asks for, and neither may be derived away. -func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) string { +func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, routed []string) string { var b strings.Builder b.WriteString("# Computed by the mesh from what is assigned to this node.\n") b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n") @@ -252,6 +252,21 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str b.WriteString("\t\ticmp type echo-request accept\n") b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n") + // **What a machine it routes for must be able to ask it** (novox/hq ADR 0137). A guest on one of + // these networks gets its address and its names from this machine, over the bridge it is on, and + // those two questions arrive at the input chain like any other. Denied, the guest never gets an + // address and never resolves a name — which is not "a closed port" but a network that does not + // work at all, and it is this machine's own guest asking. + // + // Only these ports, and only for a network that was named: everything else a guest might want + // from its host is a port somebody declares, like every other port on this machine. + for _, network := range routed { + family := saddrFamily(network) + b.WriteString("\t\t# address and name service for a network this machine routes\n") + b.WriteString(fmt.Sprintf("\t\t%s saddr %s udp dport { 53, 67 } accept\n", family, network)) + b.WriteString(fmt.Sprintf("\t\t%s saddr %s tcp dport 53 accept\n", family, network)) + } + // **ssh, always, and not because a module asked.** // // Every other line in this chain is derived from what is assigned here, which is the whole @@ -375,6 +390,13 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str b.WriteString(fmt.Sprintf("\t\t# %s\n", network.why)) b.WriteString(fmt.Sprintf("\t\tip saddr %s accept\n", network.cidr)) } + // And what this machine says it routes beyond them (novox/hq ADR 0137). Added to the defaults + // above, never replacing them: a machine that names one range has not stopped hosting whatever + // was already on the runtime's own. + for _, network := range routed { + b.WriteString("\t\t# a network this machine routes for what it hosts\n") + b.WriteString(fmt.Sprintf("\t\t%s saddr %s accept\n", saddrFamily(network), network)) + } if len(rules) > 0 { b.WriteString("\n") @@ -442,6 +464,16 @@ var runtimeNetworks = []struct{ cidr, why string }{ {"192.168.128.0/17", "the networks its compose files are given"}, } +// saddrFamily is the match a network's family is written with: `ip saddr` or `ip6 saddr`. One match +// for both families is a syntax error, and a ruleset that does not load is a machine filtering +// nothing while its service reports a fault — the same reason byFamily below exists. +func saddrFamily(network string) string { + if strings.Contains(network, ":") { + return "ip6" + } + return "ip" +} + // byFamily splits addresses into the two nftables understands separately. // // `ip saddr` and `ip6 saddr` are different matches, and one set holding both families is a syntax diff --git a/internal/catalogue/filtering_foundation_test.go b/internal/catalogue/filtering_foundation_test.go index c98da09..fb53cf3 100644 --- a/internal/catalogue/filtering_foundation_test.go +++ b/internal/catalogue/filtering_foundation_test.go @@ -19,7 +19,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { // A machine on the private network, with one ordinary module rule, and nothing that mentions // the broker — which is every machine. rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}} - out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}) + out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil) if !strings.Contains(out, "tcp dport 5671 accept") { t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out) @@ -48,7 +48,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or // a panic. A control plane in that state cannot issue tokens either, which is where it surfaces. func TestNoBrokerMeansNoFoundationRuleRatherThanNoRuleset(t *testing.T) { - out := AsNftables(nil, []string{"10.42.0.1"}, false, nil) + out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil) if !strings.Contains(out, "table inet mesh") { t.Fatalf("no ruleset at all:\n%s", out) } diff --git a/internal/catalogue/filtering_routed_test.go b/internal/catalogue/filtering_routed_test.go new file mode 100644 index 0000000..98dc618 --- /dev/null +++ b/internal/catalogue/filtering_routed_test.go @@ -0,0 +1,99 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A machine's own guests keep working when the filter it is given denies by default. +// +// **The forward chain allowed the container runtime's two default pools and nothing else** — named +// in this package's code with a comment saying a machine configured otherwise "needs this to say +// so", and no way to say it (novox/hq ADR 0137). Measured on a workstation on 2026-09-28: the flip +// to the derived filter cut egress for five of its container networks, allocated from ranges those +// two defaults do not cover, and for every network its test beds create. Nothing reported a fault. +// The guests simply could not reach anything, and the machine went on saying it had applied what it +// was told. +func TestWhatAMachineSaysItRoutesKeepsBeingForwarded(t *testing.T) { + const beds = "10.0.0.0/8" + + ruleset := AsNftables(nil, []string{"10.10.0.1"}, false, nil, []string{beds}) + + forward := chainOf(t, ruleset, "forward") + if !strings.Contains(forward, "ip saddr "+beds+" accept") { + t.Errorf("the forward chain does not accept what the machine says it routes (%s):\n%s", + beds, forward) + } + // The defaults stay. A machine that names one range has not stopped hosting whatever was + // already on the runtime's own pools, and losing those would trade one silent breakage for + // another. + for _, network := range runtimeNetworks { + if !strings.Contains(forward, "ip saddr "+network.cidr+" accept") { + t.Errorf("naming a network dropped the runtime's own %s:\n%s", network.cidr, forward) + } + } + + // And its guests can still ask this machine the two questions that make a network usable at + // all: what is my address, and what is that name. + input := chainOf(t, ruleset, "input") + for _, want := range []string{ + "ip saddr " + beds + " udp dport { 53, 67 } accept", + "ip saddr " + beds + " tcp dport 53 accept", + } { + if !strings.Contains(input, want) { + t.Errorf("the input chain is missing %q, so a guest on %s gets no address and "+ + "resolves no name:\n%s", want, beds, input) + } + } +} + +// A machine that says nothing is filtered exactly as it was before this existed. +// +// The change has to be additive on every machine already converged: novox has been carrying this +// mesh's public services behind the derived filter for weeks, and a new line in its ruleset is a +// change to a production firewall nobody asked for. +func TestAMachineThatNamesNoNetworksIsFilteredAsBefore(t *testing.T) { + rules := []Rule{{Port: 443, Protocol: "tcp", From: FromEverywhere, Because: []string{"proxy"}}} + + said := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, nil) + quiet := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, []string{}) + + if said != quiet { + t.Errorf("nil and empty render differently:\n%s\n---\n%s", said, quiet) + } + if strings.Contains(said, "a network this machine routes") { + t.Errorf("a machine that named nothing carries a line about what it routes:\n%s", said) + } +} + +// The two families are matched differently, and one set holding both is a syntax error — a ruleset +// that does not load is a machine filtering nothing while its unit reports a fault. +func TestARoutedNetworkIsMatchedInItsOwnFamily(t *testing.T) { + ruleset := AsNftables(nil, nil, false, nil, []string{"10.0.0.0/8", "fd00::/8"}) + + if !strings.Contains(ruleset, "ip saddr 10.0.0.0/8 accept") { + t.Errorf("the v4 network is not matched as ip saddr:\n%s", ruleset) + } + if !strings.Contains(ruleset, "ip6 saddr fd00::/8 accept") { + t.Errorf("the v6 network is not matched as ip6 saddr:\n%s", ruleset) + } + if strings.Contains(ruleset, "ip saddr fd00::/8") { + t.Errorf("a v6 network is matched as ip saddr, which nftables refuses:\n%s", ruleset) + } +} + +// chainOf is one chain's body, so a test about the forward chain cannot pass on a line in the input +// chain that happens to look the same. +func chainOf(t *testing.T, ruleset, name string) string { + t.Helper() + start := strings.Index(ruleset, "chain "+name+" {") + if start < 0 { + t.Fatalf("no chain %q in:\n%s", name, ruleset) + } + rest := ruleset[start:] + end := strings.Index(rest, "\n\t}") + if end < 0 { + t.Fatalf("chain %q does not end:\n%s", name, rest) + } + return rest[:end] +} diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index 22a4e3f..00e3040 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) { t.Fatalf("a module that wanted this port open is not named: %+v", rules[0]) } // The consequence, which is the reason this matters: removing web must not read as closing 443. - nft := AsNftables(rules, nil, false, nil) + nft := AsNftables(rules, nil, false, nil, nil) if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") { t.Fatalf("the rendered rule set does not name both sources:\n%s", nft) } @@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) { func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil) // Naming the chain, not just the policy: the forward chain drops too, and an assertion on // "policy drop" alone passes while the input chain accepts everything. It did, once, here. if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") { @@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { // `flush ruleset` would do the first and not the second: it empties every table on the machine, // including the ones the container runtime writes for its bridges. func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { - nft := AsNftables(nil, nil, false, nil) + nft := AsNftables(nil, nil, false, nil, nil) if strings.Contains(nft, "flush ruleset") { t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft) } @@ -160,7 +160,7 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { // So the chain exists and denies by default, and the runtime's own networks are allowed explicitly // — which is how the system being replaced has been doing it on these machines for months. func TestWhatIsForwardedIsGovernedToo(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil) if !strings.Contains(nft, "hook forward priority filter; policy drop") { t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft) } @@ -168,7 +168,7 @@ func TestWhatIsForwardedIsGovernedToo(t *testing.T) { // And containers keep working, which is the whole reason the chain was left out before. func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil) for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} { if !strings.Contains(nft, "ip saddr "+network+" accept") { t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft) @@ -183,7 +183,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil) if !strings.Contains(nft, "ct original proto-dst 8080 accept") { t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft) } @@ -193,7 +193,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2"}, false, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") { t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft) } @@ -203,7 +203,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil) + }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") { t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft) } @@ -213,7 +213,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), nil, false, nil) + }}, nil), nil, false, nil, nil) if strings.Contains(nft, "dport 5432 accept") { t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft) } @@ -226,7 +226,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { func TestAMachineScopedPortIsNotOpened(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}}, - }}, nil), []string{"198.51.100.2"}, false, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil) if strings.Contains(nft, "dport 6379 accept") { t.Fatalf("a port for this machine only was opened to the network:\n%s", nft) } @@ -268,7 +268,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) { func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil) + }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") { t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft) } @@ -672,7 +672,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) { // loading the rules lives on conntrack until it drops, and then the machine is reached from a // rescue console (novox/hq issue 047). func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil) + nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") { t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft) } @@ -685,7 +685,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { // And from outside as well, on a machine that faces outward — because that is the way in when the // private network is the thing that broke. func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil) + nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil) if !strings.Contains(nft, "\t\ttcp dport 22 accept") { t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft) } @@ -697,7 +697,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { // to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody // adopts, reached over the network, closed by the act of adopting it. func TestSSHIsNeverLeftWithoutARule(t *testing.T) { - nft := AsNftables(nil, nil, false, nil) + nft := AsNftables(nil, nil, false, nil, nil) if !strings.Contains(nft, "tcp dport 22 accept") { t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft) } diff --git a/internal/catalogue/print_rehearsal_test.go b/internal/catalogue/print_rehearsal_test.go index 65fbb58..c136ab2 100644 --- a/internal/catalogue/print_rehearsal_test.go +++ b/internal/catalogue/print_rehearsal_test.go @@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) { rules := mustFilter(t, Resolution{Modules: []Manifest{ {Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}}, }}, nil) - t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil)) + t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil)) } diff --git a/internal/inventory/migrations/0043-a-machine-says-which-networks-it-routes.sql b/internal/inventory/migrations/0043-a-machine-says-which-networks-it-routes.sql new file mode 100644 index 0000000..4372efa --- /dev/null +++ b/internal/inventory/migrations/0043-a-machine-says-which-networks-it-routes.sql @@ -0,0 +1,19 @@ +-- The networks a machine routes for what it hosts, beyond the container runtime's own defaults. +-- +-- novox/hq ADR 0137. The derived packet filter denies forwarding by default and then allows the +-- container runtime's two default pools, named in the controller's code with a comment saying that +-- a machine configured otherwise "needs this to say so" — and no way to say it. So the filter was +-- correct only on a machine whose runtime used the defaults, and silently wrong on any other. +-- +-- Measured on 2026-09-28: flipping a workstation to the derived filter cut egress for five of its +-- container networks and for every network its test beds create, because those are allocated from +-- ranges the two defaults do not cover. Nothing reported a fault; the containers simply could not +-- reach anything. +-- +-- A node-level fact, beside the node's public domain and for the same reason: it is a property of +-- the machine, not of whichever module happens to load the filter today. Swapping that module must +-- not lose it. +-- +-- Null for a machine that routes nothing but the runtime's defaults, which is the ordinary case and +-- what every machine held before this column existed. +alter table node add column routed_networks jsonb; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index b0d52b8..c166a94 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "sort" "strings" "time" @@ -518,6 +519,69 @@ func (i *Inventory) PublicDomainOf(ctx context.Context, name string) (string, er return *domain, nil } +// SetRoutedNetworks records the networks this machine routes for what it hosts, beyond the +// container runtime's own default pools. +// +// A node-level fact (novox/hq ADR 0137), beside the node's public domain: the machine routes them, +// not whichever module loads the filter, so swapping that module must not lose them. Added to the +// runtime's defaults rather than replacing them, so a machine that says one range does not lose the +// ranges its containers were already using. An empty list clears it. +// +// Each entry is checked as a CIDR here rather than at render time: an address that does not parse +// becomes a line nftables refuses, and a refused ruleset is a machine that filters nothing while +// its service reports a configuration fault. +func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks []string) error { + node, err := i.NodeByName(ctx, name) + if err != nil { + return err + } + var kept []string + for _, n := range networks { + n = strings.TrimSpace(n) + if n == "" { + continue + } + if _, _, err := net.ParseCIDR(n); err != nil { + return fmt.Errorf("%q is not a network in CIDR form (10.0.0.0/8, 192.168.0.0/16): %w", + n, err) + } + kept = append(kept, n) + } + if len(kept) == 0 { + _, err = i.store.Pool().Exec(ctx, + `update node set routed_networks = null where id = $1`, node.ID) + return err + } + body, err := json.Marshal(kept) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `update node set routed_networks = $2 where id = $1`, node.ID, string(body)) + return err +} + +// RoutedNetworksOf is the networks a machine routes for what it hosts, empty when it has named none. +func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string, error) { + var body []byte + err := i.store.Pool().QueryRow(ctx, + `select routed_networks from node where name = $1`, name).Scan(&body) + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + if err != nil { + return nil, err + } + if len(body) == 0 { + return nil, nil + } + var networks []string + if err := json.Unmarshal(body, &networks); err != nil { + return nil, fmt.Errorf("the networks recorded for %s are not a list: %w", name, err) + } + return networks, nil +} + // RecordOverlayKey keeps the public half a node generated. func (i *Inventory) RecordOverlayKey(ctx context.Context, node, key string) error { if strings.TrimSpace(key) == "" {