From fe5988c53678164a2b798116a2a5905048af424f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 28 Sep 2026 23:49:14 +0200 Subject: [PATCH] The filter constrains what arrives from outside, and names no network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forward chain blocked everything passing through the machine and then allowed the machine's own containers back by naming their address ranges: 172.16.0.0/12 and 192.168.128.0/17 fixed here, the rest recorded per machine by 0043. Every way of keeping that list correct fails — a constant describes one machine, a recorded range goes stale in silence and cannot tell a network the mesh made from one a predecessor left behind, and generating it from the modules would put half the rule set on the machine. The mesh has no position on a container reaching outward: that is not a port opened to anybody. So both chains are written around the links traffic arrives on. What did not arrive from outside is accepted in one line; what did meets the declared rules. The tunnel is named beside the outward links rather than treated as inside, or a port nothing declares would be reachable from every machine in the mesh. A machine that has not reported an outward link is sent no filter and keeps the one it has, refused where a person reads it rather than as a rule set that will not load. Removes the two constants, `node networks`, and the column behind it. novox/hq ADR 0140, superseding 0137 and 0139. --- cmd/mesh-controller/adoption.go | 58 +++---- cmd/mesh-controller/nodes.go | 78 ++-------- cmd/mesh-controller/plan.go | 10 +- internal/catalogue/adoption_test.go | 26 ++-- internal/catalogue/declaration.go | 40 ++++- internal/catalogue/filtering.go | 105 +++++++------ .../catalogue/filtering_foundation_test.go | 4 +- internal/catalogue/filtering_routed_test.go | 99 ------------ internal/catalogue/filtering_test.go | 144 +++++++++++++++--- internal/catalogue/print_rehearsal_test.go | 2 +- ...-machine-says-which-links-face-outside.sql | 26 ++++ internal/inventory/nodes.go | 59 +++---- internal/link/enrolment.go | 11 ++ internal/link/protocol.go | 14 ++ internal/link/serve.go | 3 +- 15 files changed, 347 insertions(+), 332 deletions(-) delete mode 100644 internal/catalogue/filtering_routed_test.go create mode 100644 internal/inventory/migrations/0044-a-machine-says-which-links-face-outside.sql diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 2e5bdc0..baf9da8 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -7,7 +7,6 @@ import ( "errors" "flag" "fmt" - "net" "slices" "sort" "strings" @@ -310,7 +309,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s return "", err } derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, - outward: plan.PublicDomain != "", routed: with.Routed} + outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks} preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter]) preview += "\n\n preview " + saw if !yes { @@ -415,16 +414,17 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter, b.WriteString(" not previewed: traffic the machine routes that is not a published port " + "(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " + "declares it\n") - // What it routes, said rather than left to the sentence above (novox/hq ADR 0137). The filter - // forwards the container runtime's own default pools without being told; anything else is this - // list, and a machine whose guests live outside those pools and names none of them loses their - // egress at the flip, silently, which is how this was found. - if len(derived.routed) > 0 { - b.WriteString(fmt.Sprintf(" it routes: %s — kept forwarded, and their guests keep "+ - "address and name service\n", strings.Join(derived.routed, ", "))) + // Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR + // 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which + // is what a reader most wants to know, because the previous shape of this filter cut a machine's + // guests off at the flip without saying so, and that is how this was found. + if len(derived.outwardLinks) > 0 { + b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+ + "— everything its own guests send keeps working\n", + strings.Join(derived.outwardLinks, ", "))) } else { - b.WriteString(" it routes: nothing said, so only the container runtime's own default " + - "pools are forwarded — `node networks ...` if its guests live elsewhere\n") + b.WriteString(" it has reported no link facing outside, so no filter can be composed " + + "for it — the flip is refused until it reports one\n") } isTaken := map[string]bool{} @@ -485,9 +485,10 @@ type derivedFilter struct { // mesh is every address on the private network; outward says the machine faces outside. mesh []string outward bool - // routed is the networks this machine says it routes for what it hosts (novox/hq ADR 0137): - // their guests keep address and name service, and what they send onward keeps being forwarded. - routed []string + // outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140). + // The filter constrains what arrives on them; everything arriving elsewhere is this machine's + // own guest and is not filtered. + outwardLinks []string } // closesOutside is what a narrowing from everywhere to the private network is called: it closes. @@ -513,12 +514,11 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "stays open — the mesh's own, from anywhere" } } - // A guest on a network this machine routes asks it for an address and for names, and those two - // arrive here (novox/hq ADR 0137). Matched on the listener's own address: a resolver bound to a - // bridge in one of those networks is the one its guests ask. - if within(r.Address, d.routed) && - ((r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53)) { - return "stays open — address and name service for a network this machine routes" + // This machine's own guests ask it for an address and for names, and those two arrive here + // (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an + // outward link keeps answering them. + if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) { + return "stays open — this machine's own guests asking it for an address and for names" } for _, rule := range d.rules { if rule.Port != r.Port || rule.Protocol != r.Protocol { @@ -542,24 +542,6 @@ func (d derivedFilter) fate(r inventory.Reach) string { return "WILL CLOSE — no module assigned here declares it" } -// within says whether an address the machine reported sits inside one of the networks it routes. -func within(address string, networks []string) bool { - ip := net.ParseIP(strings.Trim(address, "[]")) - if ip == nil { - return false - } - for _, n := range networks { - _, block, err := net.ParseCIDR(n) - if err != nil { - continue - } - if block.Contains(ip) { - return true - } - } - return false -} - // countReachable is how much of a node's account of itself names something off the machine. // Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it, // so a report of loopback alone says nothing about what the filter would close. diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 59aa7b8..e9cfa29 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -21,8 +21,7 @@ import ( func nodeCommand(ctx context.Context, args []string) error { if len(args) == 0 { - return errors.New("node add , node list, node show , " + publicDomainUsage + - ", or " + networksUsage) + return errors.New("node add , node list, node show , or " + publicDomainUsage) } open, err := openStores(ctx) if err != nil { @@ -68,10 +67,16 @@ func nodeCommand(ctx context.Context, args []string) error { return publicDomain(ctx, inv, args[1:]) case "networks": - // The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived - // filter must keep forwarding, beyond the container runtime's own default pools which it - // allows without being told. Reports with no argument, for the same reason the domain does. - return nodeNetworks(ctx, inv, args[1:]) + // Removed by novox/hq ADR 0140, which superseded the record that added it. The filter no + // longer names any network: it constrains what arrives from outside the machine and says + // nothing about what did not, so there is no list to keep. Answered rather than met with + // "unknown command", because this was the documented way to stop a flip cutting a machine's + // containers off and somebody will reasonably still type it. + return errors.New("`node networks` is gone (novox/hq ADR 0140). The filter constrains what " + + "arrives from outside this machine and says nothing about traffic that did not, so no " + + "network is named anywhere and nothing needs to be said to keep a machine's own " + + "containers reaching outward. The machine reports which of its links face outside; see " + + "`node show `") case "account": // The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is @@ -151,67 +156,6 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st return nil } -const networksUsage = "node networks — what it routes now; " + - " ... to set them; --clear to route only the container runtime's own" - -// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts. -// -// The same three forms as the domain above, and the read-shaped one reports rather than clearing, -// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it -// by asking a question is not a mistake anybody can see afterwards. -func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error { - set := flag.NewFlagSet("node networks", flag.ContinueOnError) - clear := set.Bool("clear", false, - "route only the container runtime's own default pools, as a machine that has said nothing does") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) == 0 { - return errors.New(networksUsage) - } - node := positionals[0] - - switch { - case *clear && len(positionals) > 1: - return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+ - "things and the mesh will not choose between them", node, strings.Join(positionals[1:], " ")) - - case *clear: - if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil { - return err - } - fmt.Printf("%s routes only the container runtime's own default pools\n", node) - fmt.Printf(" run `push %s` to send its filter\n", node) - return nil - - case len(positionals) > 1: - if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil { - return err - } - fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", ")) - fmt.Printf(" its filter forwards them, and their guests keep address and name service\n") - fmt.Printf(" run `push %s` to send it\n", node) - return nil - - default: - if _, err := inv.NodeByName(ctx, node); err != nil { - return err - } - networks, err := inv.RoutedNetworksOf(ctx, node) - if err != nil { - return err - } - if len(networks) == 0 { - fmt.Printf("%s routes only the container runtime's own default pools\n", node) - fmt.Printf(" `node networks %s ...` if its guests live elsewhere\n", node) - return nil - } - fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", ")) - return nil - } -} - const publicDomainUsage = "node public-domain — what it is now; " + " to set it; --clear to take it away" diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index a70a29f..c6f174d 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -640,9 +640,9 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } - // The networks this machine routes for what it hosts, which the derived filter must forward for - // (novox/hq ADR 0137). - routed, err := inv.RoutedNetworksOf(ctx, node) + // Which of this machine's links face outside, which is what the derived filter is written + // around (novox/hq ADR 0140). Reported by the machine, never set. + outwardLinks, err := inv.OutwardLinksOf(ctx, node) if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } @@ -651,8 +651,8 @@ func renderingFor(ctx context.Context, open *stores, node string, Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, - Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, - Kept: kept, Adopted: record.Adopted, Routed: routed, + Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation, + Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks, Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, BusUsers: busUsers, }, record, nil diff --git a/internal/catalogue/adoption_test.go b/internal/catalogue/adoption_test.go index 7f313d6..1fc931c 100644 --- a/internal/catalogue/adoption_test.go +++ b/internal/catalogue/adoption_test.go @@ -59,7 +59,11 @@ func anchorRendering(adopted bool) Rendering { Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}}, Mesh: []string{"10.42.0.1"}, Foundation: []int{5671}, - Adopted: adopted, + // What the machine reported faces outside, which every rule in the filter is written + // around (novox/hq ADR 0140). + OutwardLinks: []string{"eth0"}, + TunnelInterface: "mesh0", + Adopted: adopted, // Genesis takes the foundation's modules. Taken: map[string]bool{"postgres": true, "lavinmq": true}, } @@ -575,11 +579,13 @@ func TestAGivenMachineSideReachesTheFilterTheOpeningAndTheConsumer(t *testing.T) } r := Resolution{Node: "anchor", Modules: []Manifest{forge}} with := Rendering{ - Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)}, - Given: map[string]map[int]int{"forge": given}, - Mesh: []string{"10.77.0.1"}, - Adopted: true, - Taken: map[string]bool{"forge": true}, + Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)}, + Given: map[string]map[int]int{"forge": given}, + Mesh: []string{"10.77.0.1"}, + Adopted: true, + OutwardLinks: []string{"eth0"}, + TunnelInterface: "mesh0", + Taken: map[string]bool{"forge": true}, } // What the runtime is handed: the machine's own port on the outside, the container's within. @@ -660,9 +666,11 @@ func TestALongFormPortIsOpenedWhereTheManifestPublishesIt(t *testing.T) { forge := aForge() r := Resolution{Node: "anchor", Modules: []Manifest{forge}} composed, err := r.Compose(Rendering{ - Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)}, - Mesh: []string{"10.77.0.1"}, - Adopted: true, + Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)}, + Mesh: []string{"10.77.0.1"}, + Adopted: true, + OutwardLinks: []string{"eth0"}, + TunnelInterface: "mesh0", }) if err != nil { t.Fatal(err) diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 6a44726..5bdb970 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -124,10 +124,15 @@ type Rendering struct { // nothing on this node keeps them, or the mesh has no operator key. Kept *KeptExport - // Routed is the networks this machine routes for what it hosts, beyond the container runtime's - // own default pools, which the filter allows without being told (novox/hq ADR 0137). A node-level - // fact: the machine routes them, and the module that loads the filter may be replaced. - Routed []string + // OutwardLinks is the links this machine reported as facing outside it, which the filter is + // written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh + // composes no filter for it rather than writing a rule around a link with no name. + OutwardLinks []string + + // TunnelInterface is the interface the mesh's private network runs on, named here rather than + // imported because the overlay package rests on this one. Traffic arriving on it is the mesh's, + // not this machine's own guest, so the filter admits it only by a rule. + TunnelInterface string // Foundation is the ports the mesh itself needs reachable on every machine, which no module // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker @@ -350,7 +355,21 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri if err != nil { return nil, err } - filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation, with.Routed) + // **A machine that has not said which links face outside is sent no filter** (novox/hq ADR + // 0140). The whole chain is written around those links: with none, the rule that lets this + // machine's own guests keep working would name an empty set, which nftables refuses, and a rule + // set that does not load is a machine filtering nothing while its unit reports success. Refused + // here, where a person reads it, rather than on the machine — and the machine keeps the filter + // it already has. + if filters := r.filtersHere(); filters != "" && len(with.OutwardLinks) == 0 { + return nil, fmt.Errorf( + "%s cannot be sent a filter: it has not reported which of its links face outside, and "+ + "every rule in the chain is written around them. It reports that on each apply; "+ + "`node show %s` says whether it has. Until then %s is not sent, and the machine "+ + "keeps the filter it has", r.Node, r.Node, filters) + } + filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation, + with.OutwardLinks, with.TunnelInterface) var out []map[string]any for _, m := range r.Modules { @@ -857,6 +876,17 @@ func mapping(written string) (outer, inner int, address string, ok bool) { return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true } +// filtersHere is the module on this node that loads the machine's packet filter, or empty when none +// does. Named rather than counted: a refusal that says which module is one step from acted on. +func (r Resolution) filtersHere() string { + for _, m := range r.Modules { + if m.Filtering != nil { + return m.Module + } + } + return "" +} + // Rules is the rule set this node's filter is derived from: every module's listens, what was // computed for this machine, and each module's per-node exposure. The same answer whether the node // is adopted or converged — the one loads it as a filter, the other declares it as openings. diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index bd61b40..e246e87 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -230,7 +230,23 @@ const SSHPort = 22 // It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can // repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing // any module asks for, and neither may be derived away. -func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, routed []string) string { +func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, + outwardLinks []string, tunnel string) string { + // The links that are not this machine's own: the ones facing outside, and the mesh's tunnel. + // Traffic arriving on any of them is admitted only by a rule below; traffic arriving anywhere + // else is this machine's own guest and is not something the mesh has a position on. + // + // The tunnel is named here deliberately. Treating it as "not outside" would make a port nothing + // declares reachable from every machine in the mesh, which is the derivation abandoned. + quoted := make([]string, 0, len(outwardLinks)+1) + for _, link := range outwardLinks { + quoted = append(quoted, fmt.Sprintf("%q", link)) + } + if tunnel != "" { + quoted = append(quoted, fmt.Sprintf("%q", tunnel)) + } + inward := strings.Join(quoted, ", ") + var b strings.Builder b.WriteString("# Computed by the mesh from what is assigned to this node.\n") b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n") @@ -252,19 +268,20 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou b.WriteString("\t\ticmp type echo-request accept\n") b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n") - // **What a machine it routes for must be able to ask it** (novox/hq ADR 0137). A guest on one of - // these networks gets its address and its names from this machine, over the bridge it is on, and - // those two questions arrive at the input chain like any other. Denied, the guest never gets an - // address and never resolves a name — which is not "a closed port" but a network that does not - // work at all, and it is this machine's own guest asking. + // **What this machine's own guests must be able to ask it** (novox/hq ADR 0140). A guest gets + // its address and its names from this machine, over the link it is on, and those two questions + // arrive at the input chain like any other. Denied, the guest never gets an address and never + // resolves a name — which is not "a closed port" but a network that does not work at all, and it + // is this machine's own guest asking. // - // Only these ports, and only for a network that was named: everything else a guest might want - // from its host is a port somebody declares, like every other port on this machine. - for _, network := range routed { - family := saddrFamily(network) - b.WriteString("\t\t# address and name service for a network this machine routes\n") - b.WriteString(fmt.Sprintf("\t\t%s saddr %s udp dport { 53, 67 } accept\n", family, network)) - b.WriteString(fmt.Sprintf("\t\t%s saddr %s tcp dport 53 accept\n", family, network)) + // Asked for by the link it arrives on rather than by the address it comes from, for the reason + // the forward chain below no longer names an address: a range describes one machine and goes + // stale in silence. Anything arriving from outside, or over the tunnel, is not a guest of this + // machine and asks through a port somebody declared, like everything else. + if len(inward) > 0 { + b.WriteString("\t\t# this machine's own guests asking it for an address and for names\n") + b.WriteString(fmt.Sprintf("\t\tiifname != { %s } udp dport { 53, 67 } accept\n", inward)) + b.WriteString(fmt.Sprintf("\t\tiifname != { %s } tcp dport 53 accept\n", inward)) } // **ssh, always, and not because a module asked.** @@ -376,26 +393,36 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou // about the ports most worth protecting. Rehearsed on three machines: loading these rules // refused a port on the host and left a published container port reachable (novox/hq issue 047). // - // The way through is the one the system being replaced already used: deny by default here, and - // then explicitly allow the runtime's own networks, so containers keep working while everything - // else has to be asked for. + // **What it constrains is traffic arriving from OUTSIDE this machine, and nothing else** + // (novox/hq ADR 0140). + // + // It used to deny everything here and then allow the machine's own containers back by naming + // the address ranges they sit on — two ranges fixed in this file and the rest recorded per + // machine. Every way of keeping that list correct failed. A constant describes one machine. A + // recorded range goes stale in silence and cannot tell a network the mesh made from one a + // predecessor left behind. Generating it from the modules would have put half this rule set on + // the machine. + // + // The list should not exist, because the mesh has no position on a container reaching outward: + // that is not a port opened to anybody. So traffic that did not arrive from outside is accepted + // in one line, and what did arrive from outside is allowed only where a rule below admits it. + // + // The tunnel is not "not outside". Accepting everything off it would make a port nothing + // declares reachable from any machine in the mesh, which is the derivation abandoned — so it is + // named here beside the outward links, and traffic arriving on it meets the rules below like + // anything else. b.WriteString("\tchain forward {\n") b.WriteString("\t\ttype filter hook forward priority filter; policy drop;\n") b.WriteString("\t\tct state established,related accept\n") b.WriteString("\t\tct state invalid drop\n") b.WriteString("\n") - // What the container runtime created. Without these, denying by default stops every container - // on the machine — which is exactly the failure the absent chain was avoiding, avoided properly. - for _, network := range runtimeNetworks { - b.WriteString(fmt.Sprintf("\t\t# %s\n", network.why)) - b.WriteString(fmt.Sprintf("\t\tip saddr %s accept\n", network.cidr)) - } - // And what this machine says it routes beyond them (novox/hq ADR 0137). Added to the defaults - // above, never replacing them: a machine that names one range has not stopped hosting whatever - // was already on the runtime's own. - for _, network := range routed { - b.WriteString("\t\t# a network this machine routes for what it hosts\n") - b.WriteString(fmt.Sprintf("\t\t%s saddr %s accept\n", saddrFamily(network), network)) + // Only when there is a link to name. An empty set is a line nftables refuses, and a rule set + // that does not load is a machine filtering nothing while its unit reports success — so the + // chain denies rather than renders nonsense. Composing a declaration for a machine that has + // named none is refused upstream, so this is a floor and not a path anything travels. + if inward != "" { + b.WriteString("\t\t# this machine's own guests reaching outward: not a port opened to anybody\n") + b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward)) } if len(rules) > 0 { @@ -452,28 +479,6 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou return b.String() } -// runtimeNetworks are the container runtime's own networks, which must keep working when the -// forward chain denies by default. -// -// Taken from what the system being replaced allows, which has been carrying this machine's traffic -// for months: the runtime's bridge range and the range its compose files are given. A machine whose -// runtime is configured with something else needs this to say so — which is a thing the mesh cannot -// derive and a reason this list is named here rather than computed. -var runtimeNetworks = []struct{ cidr, why string }{ - {"172.16.0.0/12", "the container runtime's bridge networks"}, - {"192.168.128.0/17", "the networks its compose files are given"}, -} - -// saddrFamily is the match a network's family is written with: `ip saddr` or `ip6 saddr`. One match -// for both families is a syntax error, and a ruleset that does not load is a machine filtering -// nothing while its service reports a fault — the same reason byFamily below exists. -func saddrFamily(network string) string { - if strings.Contains(network, ":") { - return "ip6" - } - return "ip" -} - // byFamily splits addresses into the two nftables understands separately. // // `ip saddr` and `ip6 saddr` are different matches, and one set holding both families is a syntax diff --git a/internal/catalogue/filtering_foundation_test.go b/internal/catalogue/filtering_foundation_test.go index fb53cf3..6005651 100644 --- a/internal/catalogue/filtering_foundation_test.go +++ b/internal/catalogue/filtering_foundation_test.go @@ -19,7 +19,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { // A machine on the private network, with one ordinary module rule, and nothing that mentions // the broker — which is every machine. rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}} - out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil) + out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil, "mesh0") if !strings.Contains(out, "tcp dport 5671 accept") { t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out) @@ -48,7 +48,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { // And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or // a panic. A control plane in that state cannot issue tokens either, which is where it surfaces. func TestNoBrokerMeansNoFoundationRuleRatherThanNoRuleset(t *testing.T) { - out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil) + out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil, "mesh0") if !strings.Contains(out, "table inet mesh") { t.Fatalf("no ruleset at all:\n%s", out) } diff --git a/internal/catalogue/filtering_routed_test.go b/internal/catalogue/filtering_routed_test.go deleted file mode 100644 index 98dc618..0000000 --- a/internal/catalogue/filtering_routed_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package catalogue - -import ( - "strings" - "testing" -) - -// A machine's own guests keep working when the filter it is given denies by default. -// -// **The forward chain allowed the container runtime's two default pools and nothing else** — named -// in this package's code with a comment saying a machine configured otherwise "needs this to say -// so", and no way to say it (novox/hq ADR 0137). Measured on a workstation on 2026-09-28: the flip -// to the derived filter cut egress for five of its container networks, allocated from ranges those -// two defaults do not cover, and for every network its test beds create. Nothing reported a fault. -// The guests simply could not reach anything, and the machine went on saying it had applied what it -// was told. -func TestWhatAMachineSaysItRoutesKeepsBeingForwarded(t *testing.T) { - const beds = "10.0.0.0/8" - - ruleset := AsNftables(nil, []string{"10.10.0.1"}, false, nil, []string{beds}) - - forward := chainOf(t, ruleset, "forward") - if !strings.Contains(forward, "ip saddr "+beds+" accept") { - t.Errorf("the forward chain does not accept what the machine says it routes (%s):\n%s", - beds, forward) - } - // The defaults stay. A machine that names one range has not stopped hosting whatever was - // already on the runtime's own pools, and losing those would trade one silent breakage for - // another. - for _, network := range runtimeNetworks { - if !strings.Contains(forward, "ip saddr "+network.cidr+" accept") { - t.Errorf("naming a network dropped the runtime's own %s:\n%s", network.cidr, forward) - } - } - - // And its guests can still ask this machine the two questions that make a network usable at - // all: what is my address, and what is that name. - input := chainOf(t, ruleset, "input") - for _, want := range []string{ - "ip saddr " + beds + " udp dport { 53, 67 } accept", - "ip saddr " + beds + " tcp dport 53 accept", - } { - if !strings.Contains(input, want) { - t.Errorf("the input chain is missing %q, so a guest on %s gets no address and "+ - "resolves no name:\n%s", want, beds, input) - } - } -} - -// A machine that says nothing is filtered exactly as it was before this existed. -// -// The change has to be additive on every machine already converged: novox has been carrying this -// mesh's public services behind the derived filter for weeks, and a new line in its ruleset is a -// change to a production firewall nobody asked for. -func TestAMachineThatNamesNoNetworksIsFilteredAsBefore(t *testing.T) { - rules := []Rule{{Port: 443, Protocol: "tcp", From: FromEverywhere, Because: []string{"proxy"}}} - - said := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, nil) - quiet := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, []string{}) - - if said != quiet { - t.Errorf("nil and empty render differently:\n%s\n---\n%s", said, quiet) - } - if strings.Contains(said, "a network this machine routes") { - t.Errorf("a machine that named nothing carries a line about what it routes:\n%s", said) - } -} - -// The two families are matched differently, and one set holding both is a syntax error — a ruleset -// that does not load is a machine filtering nothing while its unit reports a fault. -func TestARoutedNetworkIsMatchedInItsOwnFamily(t *testing.T) { - ruleset := AsNftables(nil, nil, false, nil, []string{"10.0.0.0/8", "fd00::/8"}) - - if !strings.Contains(ruleset, "ip saddr 10.0.0.0/8 accept") { - t.Errorf("the v4 network is not matched as ip saddr:\n%s", ruleset) - } - if !strings.Contains(ruleset, "ip6 saddr fd00::/8 accept") { - t.Errorf("the v6 network is not matched as ip6 saddr:\n%s", ruleset) - } - if strings.Contains(ruleset, "ip saddr fd00::/8") { - t.Errorf("a v6 network is matched as ip saddr, which nftables refuses:\n%s", ruleset) - } -} - -// chainOf is one chain's body, so a test about the forward chain cannot pass on a line in the input -// chain that happens to look the same. -func chainOf(t *testing.T, ruleset, name string) string { - t.Helper() - start := strings.Index(ruleset, "chain "+name+" {") - if start < 0 { - t.Fatalf("no chain %q in:\n%s", name, ruleset) - } - rest := ruleset[start:] - end := strings.Index(rest, "\n\t}") - if end < 0 { - t.Fatalf("chain %q does not end:\n%s", name, rest) - } - return rest[:end] -} diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index 00e3040..8f25d70 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) { t.Fatalf("a module that wanted this port open is not named: %+v", rules[0]) } // The consequence, which is the reason this matters: removing web must not read as closing 443. - nft := AsNftables(rules, nil, false, nil, nil) + nft := AsNftables(rules, nil, false, nil, nil, "mesh0") if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") { t.Fatalf("the rendered rule set does not name both sources:\n%s", nft) } @@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) { func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false, nil, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0") // Naming the chain, not just the policy: the forward chain drops too, and an assertion on // "policy drop" alone passes while the input chain accepts everything. It did, once, here. if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") { @@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { // `flush ruleset` would do the first and not the second: it empties every table on the machine, // including the ones the container runtime writes for its bridges. func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { - nft := AsNftables(nil, nil, false, nil, nil) + nft := AsNftables(nil, nil, false, nil, nil, "mesh0") if strings.Contains(nft, "flush ruleset") { t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft) } @@ -160,22 +160,122 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { // So the chain exists and denies by default, and the runtime's own networks are allowed explicitly // — which is how the system being replaced has been doing it on these machines for months. func TestWhatIsForwardedIsGovernedToo(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "hook forward priority filter; policy drop") { t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft) } } -// And containers keep working, which is the whole reason the chain was left out before. -func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil) - for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} { - if !strings.Contains(nft, "ip saddr "+network+" accept") { - t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft) +// And this machine's own guests keep working, which is the whole reason the chain was left out +// before — by not being mentioned (novox/hq ADR 0140). +// +// It used to be done by naming the address ranges they sit on: two fixed here and the rest recorded +// per machine. That list broke a workstation's containers at a flip and could not be made correct, +// because a range describes one machine and cannot tell a network the mesh made from one a +// predecessor left behind. What replaced it is a single line about the links traffic arrives on. +func TestThisMachinesOwnGuestsKeepWorkingWithoutBeingNamed(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0") + if !strings.Contains(nft, `iifname != { "eth0", "mesh0" } accept`) { + t.Fatalf("what did not arrive from outside is not accepted, so this machine's own guests "+ + "reach nothing:\n%s", nft) + } +} + +// No address of a machine's own networks appears anywhere in a rendered filter. +// +// This is the assertion that fails against the previous behaviour, and it is why it is written on +// the text rather than on an outcome: the two ranges were a constant in this file, so nothing but +// reading the output catches one creeping back in. +func TestNoNetworkOfTheMachinesOwnIsNamed(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0") + for _, gone := range []string{"172.16.0.0/12", "192.168.128.0/17", "saddr 192.168", "saddr 172."} { + if strings.Contains(nft, gone) { + t.Fatalf("%q is named, and a range describes one machine and goes stale in silence:\n%s", + gone, nft) } } } +// **The tunnel is constrained, not treated as inside.** +// +// Accepting everything arriving over the private network would make a port nothing declares +// reachable from every machine in the mesh — the derivation abandoned, and a rule that reads as a +// restriction while restricting nothing. So the tunnel is named beside the outward links, and +// traffic arriving on it meets the declared rules like anything else. +func TestTheTunnelIsConstrainedLikeAnOutwardLink(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0") + line := `iifname != { "eth0", "mesh0" } accept` + if !strings.Contains(nft, line) { + t.Fatalf("the tunnel is not constrained, so an undeclared port is reachable from any "+ + "machine in the mesh:\n%s", nft) + } +} + +// A machine with two links facing outside has both constrained. Asserted on the one line, because a +// rule covering one and not the other would leave a machine filtering half of what reaches it. +func TestEveryOutwardLinkIsConstrained(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0", "wlan0"}, "mesh0") + if !strings.Contains(nft, `iifname != { "eth0", "wlan0", "mesh0" } accept`) { + t.Fatalf("not every outward link is constrained:\n%s", nft) + } +} + +// A guest asks its host for an address and for names, and those two arrive at the input chain. Asked +// for by the link they arrive on, so a resolver bound anywhere but an outward link keeps answering. +func TestGuestsMayAskTheirHostForAnAddressAndNames(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0") + for _, want := range []string{ + `iifname != { "eth0", "mesh0" } udp dport { 53, 67 } accept`, + `iifname != { "eth0", "mesh0" } tcp dport 53 accept`, + } { + if !strings.Contains(nft, want) { + t.Fatalf("a guest cannot ask its host for an address or a name, which is not a closed "+ + "port but a network that does not work:\n%s", nft) + } + } +} + +// With no link named at all the chain denies rather than rendering an empty set, which nftables +// refuses — and a rule set that does not load is a machine filtering nothing while its unit reports +// success. Composing a declaration for such a machine is refused upstream; this is the floor. +func TestNoLinkNamedRendersNoCatchAllRatherThanAnEmptySet(t *testing.T) { + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil, "") + if strings.Contains(nft, "{ }") || strings.Contains(nft, "iifname != {}") { + t.Fatalf("an empty set is rendered, which nftables refuses:\n%s", nft) + } + if !strings.Contains(nft, "hook forward priority filter; policy drop") { + t.Fatalf("the forward chain does not deny:\n%s", nft) + } +} + +// A machine that has not said which links face outside is sent no filter, and the refusal names the +// module that would have loaded it so the reader knows what is being withheld. +func TestAMachineThatNamedNoOutwardLinkIsSentNoFilter(t *testing.T) { + r := Resolution{Node: "anchor", Modules: []Manifest{ + {Module: "nftables", Filtering: &Filtering{Into: "/etc/mesh/filter.nft"}}, + {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, + }} + _, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}, TunnelInterface: "mesh0"}) + if err == nil { + t.Fatal("a machine that named no outward link was sent a filter written around none") + } + for _, want := range []string{"anchor", "nftables", "face outside"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q: %v", want, err) + } + } +} + +// And a machine that names none but loads no filter is not refused: there is nothing to write. +func TestAMachineWithNoFilterModuleIsNotRefused(t *testing.T) { + r := Resolution{Node: "anchor", Modules: []Manifest{ + {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, + }} + if _, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}}); err != nil { + t.Fatalf("a machine that loads no filter was refused one: %v", err) + } +} + // A published port is matched by what the client asked for, not by where the packet ends up. // // The runtime rewrites the destination before this chain sees it, so a rule naming the published @@ -183,7 +283,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false, nil, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "ct original proto-dst 8080 accept") { t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft) } @@ -193,7 +293,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2"}, false, nil, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") { t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft) } @@ -203,7 +303,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil) + }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") { t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft) } @@ -213,7 +313,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), nil, false, nil, nil) + }}, nil), nil, false, nil, nil, "mesh0") if strings.Contains(nft, "dport 5432 accept") { t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft) } @@ -226,7 +326,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { func TestAMachineScopedPortIsNotOpened(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}}, - }}, nil), []string{"198.51.100.2"}, false, nil, nil) + }}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0") if strings.Contains(nft, "dport 6379 accept") { t.Fatalf("a port for this machine only was opened to the network:\n%s", nft) } @@ -238,7 +338,8 @@ func TestTheModuleAskingForTheRuleSetGetsEveryModulesPorts(t *testing.T) { {Module: "firewall", Filtering: &Filtering{Into: "/etc/mesh/filter.nft"}}, {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, }} - out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}}) + out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}, + OutwardLinks: []string{"eth0"}, TunnelInterface: "mesh0"}) if err != nil { t.Fatalf("declaration: %v", err) } @@ -268,7 +369,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) { func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil) + }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") { t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft) } @@ -296,7 +397,8 @@ func TestWhatTheMeshComputesIsAppliedBeforeWhatTheModuleDeclared(t *testing.T) { "restart-on": []any{"filtering"}}, }, }}} - out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}}) + out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}, + OutwardLinks: []string{"eth0"}, TunnelInterface: "mesh0"}) if err != nil { t.Fatalf("declaration: %v", err) } @@ -672,7 +774,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) { // loading the rules lives on conntrack until it drops, and then the machine is reached from a // rescue console (novox/hq issue 047). func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil) + nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil, "mesh0") if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") { t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft) } @@ -685,7 +787,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { // And from outside as well, on a machine that faces outward — because that is the way in when the // private network is the thing that broke. func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil) + nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil, "mesh0") if !strings.Contains(nft, "\t\ttcp dport 22 accept") { t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft) } @@ -697,7 +799,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { // to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody // adopts, reached over the network, closed by the act of adopting it. func TestSSHIsNeverLeftWithoutARule(t *testing.T) { - nft := AsNftables(nil, nil, false, nil, nil) + nft := AsNftables(nil, nil, false, nil, nil, "mesh0") if !strings.Contains(nft, "tcp dport 22 accept") { t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft) } diff --git a/internal/catalogue/print_rehearsal_test.go b/internal/catalogue/print_rehearsal_test.go index c136ab2..d2aa7ef 100644 --- a/internal/catalogue/print_rehearsal_test.go +++ b/internal/catalogue/print_rehearsal_test.go @@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) { rules := mustFilter(t, Resolution{Modules: []Manifest{ {Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}}, }}, nil) - t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil)) + t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil, "mesh0")) } diff --git a/internal/inventory/migrations/0044-a-machine-says-which-links-face-outside.sql b/internal/inventory/migrations/0044-a-machine-says-which-links-face-outside.sql new file mode 100644 index 0000000..457652b --- /dev/null +++ b/internal/inventory/migrations/0044-a-machine-says-which-links-face-outside.sql @@ -0,0 +1,26 @@ +-- Which of a machine's links face outside it, replacing the networks it was told to say it routes. +-- +-- novox/hq ADR 0140, superseding 0137 and 0139. The derived filter blocked everything passing +-- through a machine and then allowed the machine's own containers back by naming the address ranges +-- they sit on: two ranges fixed in the controller's source, the rest recorded by 0043's column. +-- +-- Every route to a correct list fails. A constant describes one machine. A recorded range goes stale +-- in silence, and cannot tell a network the mesh made from one a predecessor left behind — measured +-- on the control-node, where six ranges fall outside the constants and two of the six belong to +-- services the mesh does not run. Generating the list from the modules put half the rule set on the +-- machine. +-- +-- The list should not exist, because the mesh has no position on a container reaching outward: that +-- is not a port opened to anybody. The filter constrains what arrives from OUTSIDE the machine and +-- says nothing about what did not, which needs one fact instead of a list — which links "outside" +-- arrives on. +-- +-- Reported by the machine on every apply, never recorded by hand, so it cannot go stale. Null for a +-- machine that has not reported yet; the mesh composes no filter for such a machine and leaves the +-- one it has, because a rule written around a link with no name is a rule set that does not load. +alter table node add column outward_links jsonb; + +-- What 0043 recorded is not migrated into it. The ranges answered a question that no longer exists, +-- and every machine that named one keeps working without it: the traffic those ranges allowed is now +-- allowed by not having arrived from outside. +alter table node drop column routed_networks; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index c166a94..e4dd752 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "net" "sort" "strings" "time" @@ -519,37 +518,28 @@ func (i *Inventory) PublicDomainOf(ctx context.Context, name string) (string, er return *domain, nil } -// SetRoutedNetworks records the networks this machine routes for what it hosts, beyond the -// container runtime's own default pools. +// RecordOutwardLinks keeps the links a machine reported as facing outside it. // -// A node-level fact (novox/hq ADR 0137), beside the node's public domain: the machine routes them, -// not whichever module loads the filter, so swapping that module must not lose them. Added to the -// runtime's defaults rather than replacing them, so a machine that says one range does not lose the -// ranges its containers were already using. An empty list clears it. +// A reported fact, not a setting (novox/hq ADR 0140). It replaces the networks a machine used to be +// told to say it routes: the filter blocked everything passing through and then allowed the machine's +// own containers back by naming their address ranges, and every way of keeping that list correct +// failed — a constant describes one machine, and a recorded range goes stale in silence. The filter +// now constrains what arrives from outside and says nothing about what did not, and the one thing it +// needs is which links "outside" arrives on. The machine reads that from its own routing table on +// every apply, so it cannot go stale and nobody types it. // -// Each entry is checked as a CIDR here rather than at render time: an address that does not parse -// becomes a line nftables refuses, and a refused ruleset is a machine that filters nothing while -// its service reports a configuration fault. -func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks []string) error { - node, err := i.NodeByName(ctx, name) - if err != nil { - return err - } +// An empty list clears it, which is what a machine with no route off itself reports. The mesh then +// composes no filter for that machine at all. +func (i *Inventory) RecordOutwardLinks(ctx context.Context, id string, links []string) error { var kept []string - for _, n := range networks { - n = strings.TrimSpace(n) - if n == "" { - continue + for _, name := range links { + if name = strings.TrimSpace(name); name != "" { + kept = append(kept, name) } - if _, _, err := net.ParseCIDR(n); err != nil { - return fmt.Errorf("%q is not a network in CIDR form (10.0.0.0/8, 192.168.0.0/16): %w", - n, err) - } - kept = append(kept, n) } if len(kept) == 0 { - _, err = i.store.Pool().Exec(ctx, - `update node set routed_networks = null where id = $1`, node.ID) + _, err := i.store.Pool().Exec(ctx, + `update node set outward_links = null where id = $1`, id) return err } body, err := json.Marshal(kept) @@ -557,15 +547,16 @@ func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks return err } _, err = i.store.Pool().Exec(ctx, - `update node set routed_networks = $2 where id = $1`, node.ID, string(body)) + `update node set outward_links = $2 where id = $1`, id, string(body)) return err } -// RoutedNetworksOf is the networks a machine routes for what it hosts, empty when it has named none. -func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string, error) { +// OutwardLinksOf is the links a machine reported as facing outside it, empty when it has reported +// none — which is a machine the mesh composes no filter for. +func (i *Inventory) OutwardLinksOf(ctx context.Context, name string) ([]string, error) { var body []byte err := i.store.Pool().QueryRow(ctx, - `select routed_networks from node where name = $1`, name).Scan(&body) + `select outward_links from node where name = $1`, name).Scan(&body) if errors.Is(err, pgx.ErrNoRows) { return nil, fmt.Errorf("%w: %s", ErrNoSuchNode, name) } @@ -575,11 +566,11 @@ func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string if len(body) == 0 { return nil, nil } - var networks []string - if err := json.Unmarshal(body, &networks); err != nil { - return nil, fmt.Errorf("the networks recorded for %s are not a list: %w", name, err) + var links []string + if err := json.Unmarshal(body, &links); err != nil { + return nil, fmt.Errorf("the outward links recorded for %s are not a list: %w", name, err) } - return networks, nil + return links, nil } // RecordOverlayKey keeps the public half a node generated. diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 3193d71..81e06fa 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -301,6 +301,17 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err return false, err } } + // Which of its links face outside (novox/hq ADR 0140), whenever it says so. Recorded on every + // report that carries it, adopted or converged, because the filter the mesh composes is written + // around it — and never cleared by a report that carries none, which is every bare word that the + // node is there. A machine whose routing table it could not read reports nothing rather than + // guessing, and keeps whatever it last said; a machine with genuinely no route off itself is one + // the mesh composes no filter for at all. + if len(report.Outward) > 0 { + if err := e.Inventory.RecordOutwardLinks(ctx, node.ID, report.Outward); err != nil { + return false, err + } + } // What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it. if report.Tunnel != nil { if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{ diff --git a/internal/link/protocol.go b/internal/link/protocol.go index e8aef82..4d74275 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -170,6 +170,20 @@ type Report struct { // Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that // was never asked, which is every converged one. Firewall string `json:"firewall,omitempty"` + + // Outward is the links on this machine that face outside it — the ones carrying a default route + // (novox/hq ADR 0140). Every node reports it, adopted or converged, because the filter the mesh + // composes for it is written around these and nothing else. + // + // **It replaces a list of addresses.** The filter used to block everything passing through the + // machine and then allow the machine's own containers back by naming the ranges they sit on. A + // range describes one machine and goes stale in silence; the link carrying the default route is + // read afresh on every report and does not change when a module is added or removed. + // + // Empty means the machine has not said. The mesh composes no filter for such a machine and + // leaves the one it has: a rule written around a link with no name is a rule set that does not + // load, and that is a machine filtering nothing while its unit reports success. + Outward []string `json:"outward,omitempty"` // Reachable is what can be reached on the machine now: every listening socket and every // published container port. Only an adopted node reports it; it is what converging previews. Reachable []Reach `json:"reachable,omitempty"` diff --git a/internal/link/serve.go b/internal/link/serve.go index 84e438b..1099873 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -346,7 +346,8 @@ func (s *Server) reported(ctx context.Context, m Control) { // whenever it arrives, which is the behaviour the mesh has had all along. func staleAgainst(report Report) string { if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 || - report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 { + report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 || + len(report.Outward) > 0 { return "" } return report.Declared