A machine says which networks it routes, and its filter forwards them
The derived filter denies forwarding by default and then allows the container runtime's two default pools, named in this code with a comment saying a machine configured otherwise needs to say so -- and no way to say it. So the filter was right on a machine using the defaults and silently wrong on any other. Measured today: flipping a workstation to the derived filter cut egress for five of its container networks and for every network its test beds create, because those come from ranges the defaults do not cover. Nothing reported a fault; the guests just could not reach anything, while the machine reported it had applied what it was told. A node-level fact beside the public domain, because the machine routes them and the module that loads the filter may be replaced. Added to the defaults, never replacing them. Their guests also keep address and name service, without which a network does not work at all, and the converge preview now says what a machine routes instead of leaving it to a sentence about what it cannot preview.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -309,7 +310,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != ""}
|
||||
outward: plan.PublicDomain != "", routed: with.Routed}
|
||||
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||
preview += "\n\n preview " + saw
|
||||
if !yes {
|
||||
@@ -414,6 +415,17 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
||||
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
||||
"declares it\n")
|
||||
// What it routes, said rather than left to the sentence above (novox/hq ADR 0137). The filter
|
||||
// forwards the container runtime's own default pools without being told; anything else is this
|
||||
// list, and a machine whose guests live outside those pools and names none of them loses their
|
||||
// egress at the flip, silently, which is how this was found.
|
||||
if len(derived.routed) > 0 {
|
||||
b.WriteString(fmt.Sprintf(" it routes: %s — kept forwarded, and their guests keep "+
|
||||
"address and name service\n", strings.Join(derived.routed, ", ")))
|
||||
} else {
|
||||
b.WriteString(" it routes: nothing said, so only the container runtime's own default " +
|
||||
"pools are forwarded — `node networks <node> <cidr>...` if its guests live elsewhere\n")
|
||||
}
|
||||
|
||||
isTaken := map[string]bool{}
|
||||
for _, m := range taken {
|
||||
@@ -473,6 +485,9 @@ type derivedFilter struct {
|
||||
// mesh is every address on the private network; outward says the machine faces outside.
|
||||
mesh []string
|
||||
outward bool
|
||||
// routed is the networks this machine says it routes for what it hosts (novox/hq ADR 0137):
|
||||
// their guests keep address and name service, and what they send onward keeps being forwarded.
|
||||
routed []string
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
@@ -498,6 +513,13 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
return "stays open — the mesh's own, from anywhere"
|
||||
}
|
||||
}
|
||||
// A guest on a network this machine routes asks it for an address and for names, and those two
|
||||
// arrive here (novox/hq ADR 0137). Matched on the listener's own address: a resolver bound to a
|
||||
// bridge in one of those networks is the one its guests ask.
|
||||
if within(r.Address, d.routed) &&
|
||||
((r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53)) {
|
||||
return "stays open — address and name service for a network this machine routes"
|
||||
}
|
||||
for _, rule := range d.rules {
|
||||
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
||||
continue
|
||||
@@ -520,6 +542,24 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
return "WILL CLOSE — no module assigned here declares it"
|
||||
}
|
||||
|
||||
// within says whether an address the machine reported sits inside one of the networks it routes.
|
||||
func within(address string, networks []string) bool {
|
||||
ip := net.ParseIP(strings.Trim(address, "[]"))
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
for _, n := range networks {
|
||||
_, block, err := net.ParseCIDR(n)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if block.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// countReachable is how much of a node's account of itself names something off the machine.
|
||||
// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
|
||||
// so a report of loopback alone says nothing about what the filter would close.
|
||||
|
||||
@@ -148,6 +148,9 @@ func usage() {
|
||||
node public-domain <name> the domain it composes its routed names under
|
||||
node public-domain <name> <d> ...set it to d
|
||||
node public-domain <name> --clear ...it faces the outside no longer
|
||||
node networks <name> the networks it routes for what it hosts
|
||||
node networks <name> <cidr>... ...set them; its filter forwards these too
|
||||
node networks <name> --clear ...only the container runtime's own
|
||||
token issue --node <name> a one-time right to join, for an existing record
|
||||
token issue --new <name> create the record and issue for it
|
||||
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||
|
||||
@@ -21,7 +21,8 @@ import (
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage +
|
||||
", or " + networksUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -66,6 +67,12 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
// because the damage is already done by the time it prints.
|
||||
return publicDomain(ctx, inv, args[1:])
|
||||
|
||||
case "networks":
|
||||
// The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived
|
||||
// filter must keep forwarding, beyond the container runtime's own default pools which it
|
||||
// allows without being told. Reports with no argument, for the same reason the domain does.
|
||||
return nodeNetworks(ctx, inv, args[1:])
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
@@ -144,6 +151,67 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st
|
||||
return nil
|
||||
}
|
||||
|
||||
const networksUsage = "node networks <name> — what it routes now; " +
|
||||
"<name> <cidr>... to set them; <name> --clear to route only the container runtime's own"
|
||||
|
||||
// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts.
|
||||
//
|
||||
// The same three forms as the domain above, and the read-shaped one reports rather than clearing,
|
||||
// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it
|
||||
// by asking a question is not a mistake anybody can see afterwards.
|
||||
func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node networks", flag.ContinueOnError)
|
||||
clear := set.Bool("clear", false,
|
||||
"route only the container runtime's own default pools, as a machine that has said nothing does")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 {
|
||||
return errors.New(networksUsage)
|
||||
}
|
||||
node := positionals[0]
|
||||
|
||||
switch {
|
||||
case *clear && len(positionals) > 1:
|
||||
return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+
|
||||
"things and the mesh will not choose between them", node, strings.Join(positionals[1:], " "))
|
||||
|
||||
case *clear:
|
||||
if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
|
||||
fmt.Printf(" run `push %s` to send its filter\n", node)
|
||||
return nil
|
||||
|
||||
case len(positionals) > 1:
|
||||
if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", "))
|
||||
fmt.Printf(" its filter forwards them, and their guests keep address and name service\n")
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
|
||||
default:
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
networks, err := inv.RoutedNetworksOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(networks) == 0 {
|
||||
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
|
||||
fmt.Printf(" `node networks %s <cidr>...` if its guests live elsewhere\n", node)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", "))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
|
||||
@@ -640,13 +640,19 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// The networks this machine routes for what it hosts, which the derived filter must forward for
|
||||
// (novox/hq ADR 0137).
|
||||
routed, err := inv.RoutedNetworksOf(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted,
|
||||
Kept: kept, Adopted: record.Adopted, Routed: routed,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
}, record, nil
|
||||
|
||||
@@ -124,6 +124,11 @@ type Rendering struct {
|
||||
// nothing on this node keeps them, or the mesh has no operator key.
|
||||
Kept *KeptExport
|
||||
|
||||
// Routed is the networks this machine routes for what it hosts, beyond the container runtime's
|
||||
// own default pools, which the filter allows without being told (novox/hq ADR 0137). A node-level
|
||||
// fact: the machine routes them, and the module that loads the filter may be replaced.
|
||||
Routed []string
|
||||
|
||||
// Foundation is the ports the mesh itself needs reachable on every machine, which no module
|
||||
// declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker
|
||||
// is the one that matters: a machine dials it to enrol, and a firewall derived only from
|
||||
@@ -345,7 +350,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation)
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation, with.Routed)
|
||||
|
||||
var out []map[string]any
|
||||
for _, m := range r.Modules {
|
||||
|
||||
@@ -230,7 +230,7 @@ const SSHPort = 22
|
||||
// It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can
|
||||
// repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing
|
||||
// any module asks for, and neither may be derived away.
|
||||
func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) string {
|
||||
func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, routed []string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Computed by the mesh from what is assigned to this node.\n")
|
||||
b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n")
|
||||
@@ -252,6 +252,21 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str
|
||||
b.WriteString("\t\ticmp type echo-request accept\n")
|
||||
b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n")
|
||||
|
||||
// **What a machine it routes for must be able to ask it** (novox/hq ADR 0137). A guest on one of
|
||||
// these networks gets its address and its names from this machine, over the bridge it is on, and
|
||||
// those two questions arrive at the input chain like any other. Denied, the guest never gets an
|
||||
// address and never resolves a name — which is not "a closed port" but a network that does not
|
||||
// work at all, and it is this machine's own guest asking.
|
||||
//
|
||||
// Only these ports, and only for a network that was named: everything else a guest might want
|
||||
// from its host is a port somebody declares, like every other port on this machine.
|
||||
for _, network := range routed {
|
||||
family := saddrFamily(network)
|
||||
b.WriteString("\t\t# address and name service for a network this machine routes\n")
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s udp dport { 53, 67 } accept\n", family, network))
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s tcp dport 53 accept\n", family, network))
|
||||
}
|
||||
|
||||
// **ssh, always, and not because a module asked.**
|
||||
//
|
||||
// Every other line in this chain is derived from what is assigned here, which is the whole
|
||||
@@ -375,6 +390,13 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int) str
|
||||
b.WriteString(fmt.Sprintf("\t\t# %s\n", network.why))
|
||||
b.WriteString(fmt.Sprintf("\t\tip saddr %s accept\n", network.cidr))
|
||||
}
|
||||
// And what this machine says it routes beyond them (novox/hq ADR 0137). Added to the defaults
|
||||
// above, never replacing them: a machine that names one range has not stopped hosting whatever
|
||||
// was already on the runtime's own.
|
||||
for _, network := range routed {
|
||||
b.WriteString("\t\t# a network this machine routes for what it hosts\n")
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s accept\n", saddrFamily(network), network))
|
||||
}
|
||||
|
||||
if len(rules) > 0 {
|
||||
b.WriteString("\n")
|
||||
@@ -442,6 +464,16 @@ var runtimeNetworks = []struct{ cidr, why string }{
|
||||
{"192.168.128.0/17", "the networks its compose files are given"},
|
||||
}
|
||||
|
||||
// saddrFamily is the match a network's family is written with: `ip saddr` or `ip6 saddr`. One match
|
||||
// for both families is a syntax error, and a ruleset that does not load is a machine filtering
|
||||
// nothing while its service reports a fault — the same reason byFamily below exists.
|
||||
func saddrFamily(network string) string {
|
||||
if strings.Contains(network, ":") {
|
||||
return "ip6"
|
||||
}
|
||||
return "ip"
|
||||
}
|
||||
|
||||
// byFamily splits addresses into the two nftables understands separately.
|
||||
//
|
||||
// `ip saddr` and `ip6 saddr` are different matches, and one set holding both families is a syntax
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
|
||||
// A machine on the private network, with one ordinary module rule, and nothing that mentions
|
||||
// the broker — which is every machine.
|
||||
rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}}
|
||||
out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort})
|
||||
out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil)
|
||||
|
||||
if !strings.Contains(out, "tcp dport 5671 accept") {
|
||||
t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out)
|
||||
@@ -48,7 +48,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
|
||||
// And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or
|
||||
// a panic. A control plane in that state cannot issue tokens either, which is where it surfaces.
|
||||
func TestNoBrokerMeansNoFoundationRuleRatherThanNoRuleset(t *testing.T) {
|
||||
out := AsNftables(nil, []string{"10.42.0.1"}, false, nil)
|
||||
out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil)
|
||||
if !strings.Contains(out, "table inet mesh") {
|
||||
t.Fatalf("no ruleset at all:\n%s", out)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A machine's own guests keep working when the filter it is given denies by default.
|
||||
//
|
||||
// **The forward chain allowed the container runtime's two default pools and nothing else** — named
|
||||
// in this package's code with a comment saying a machine configured otherwise "needs this to say
|
||||
// so", and no way to say it (novox/hq ADR 0137). Measured on a workstation on 2026-09-28: the flip
|
||||
// to the derived filter cut egress for five of its container networks, allocated from ranges those
|
||||
// two defaults do not cover, and for every network its test beds create. Nothing reported a fault.
|
||||
// The guests simply could not reach anything, and the machine went on saying it had applied what it
|
||||
// was told.
|
||||
func TestWhatAMachineSaysItRoutesKeepsBeingForwarded(t *testing.T) {
|
||||
const beds = "10.0.0.0/8"
|
||||
|
||||
ruleset := AsNftables(nil, []string{"10.10.0.1"}, false, nil, []string{beds})
|
||||
|
||||
forward := chainOf(t, ruleset, "forward")
|
||||
if !strings.Contains(forward, "ip saddr "+beds+" accept") {
|
||||
t.Errorf("the forward chain does not accept what the machine says it routes (%s):\n%s",
|
||||
beds, forward)
|
||||
}
|
||||
// The defaults stay. A machine that names one range has not stopped hosting whatever was
|
||||
// already on the runtime's own pools, and losing those would trade one silent breakage for
|
||||
// another.
|
||||
for _, network := range runtimeNetworks {
|
||||
if !strings.Contains(forward, "ip saddr "+network.cidr+" accept") {
|
||||
t.Errorf("naming a network dropped the runtime's own %s:\n%s", network.cidr, forward)
|
||||
}
|
||||
}
|
||||
|
||||
// And its guests can still ask this machine the two questions that make a network usable at
|
||||
// all: what is my address, and what is that name.
|
||||
input := chainOf(t, ruleset, "input")
|
||||
for _, want := range []string{
|
||||
"ip saddr " + beds + " udp dport { 53, 67 } accept",
|
||||
"ip saddr " + beds + " tcp dport 53 accept",
|
||||
} {
|
||||
if !strings.Contains(input, want) {
|
||||
t.Errorf("the input chain is missing %q, so a guest on %s gets no address and "+
|
||||
"resolves no name:\n%s", want, beds, input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that says nothing is filtered exactly as it was before this existed.
|
||||
//
|
||||
// The change has to be additive on every machine already converged: novox has been carrying this
|
||||
// mesh's public services behind the derived filter for weeks, and a new line in its ruleset is a
|
||||
// change to a production firewall nobody asked for.
|
||||
func TestAMachineThatNamesNoNetworksIsFilteredAsBefore(t *testing.T) {
|
||||
rules := []Rule{{Port: 443, Protocol: "tcp", From: FromEverywhere, Because: []string{"proxy"}}}
|
||||
|
||||
said := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, nil)
|
||||
quiet := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, []string{})
|
||||
|
||||
if said != quiet {
|
||||
t.Errorf("nil and empty render differently:\n%s\n---\n%s", said, quiet)
|
||||
}
|
||||
if strings.Contains(said, "a network this machine routes") {
|
||||
t.Errorf("a machine that named nothing carries a line about what it routes:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The two families are matched differently, and one set holding both is a syntax error — a ruleset
|
||||
// that does not load is a machine filtering nothing while its unit reports a fault.
|
||||
func TestARoutedNetworkIsMatchedInItsOwnFamily(t *testing.T) {
|
||||
ruleset := AsNftables(nil, nil, false, nil, []string{"10.0.0.0/8", "fd00::/8"})
|
||||
|
||||
if !strings.Contains(ruleset, "ip saddr 10.0.0.0/8 accept") {
|
||||
t.Errorf("the v4 network is not matched as ip saddr:\n%s", ruleset)
|
||||
}
|
||||
if !strings.Contains(ruleset, "ip6 saddr fd00::/8 accept") {
|
||||
t.Errorf("the v6 network is not matched as ip6 saddr:\n%s", ruleset)
|
||||
}
|
||||
if strings.Contains(ruleset, "ip saddr fd00::/8") {
|
||||
t.Errorf("a v6 network is matched as ip saddr, which nftables refuses:\n%s", ruleset)
|
||||
}
|
||||
}
|
||||
|
||||
// chainOf is one chain's body, so a test about the forward chain cannot pass on a line in the input
|
||||
// chain that happens to look the same.
|
||||
func chainOf(t *testing.T, ruleset, name string) string {
|
||||
t.Helper()
|
||||
start := strings.Index(ruleset, "chain "+name+" {")
|
||||
if start < 0 {
|
||||
t.Fatalf("no chain %q in:\n%s", name, ruleset)
|
||||
}
|
||||
rest := ruleset[start:]
|
||||
end := strings.Index(rest, "\n\t}")
|
||||
if end < 0 {
|
||||
t.Fatalf("chain %q does not end:\n%s", name, rest)
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) {
|
||||
t.Fatalf("a module that wanted this port open is not named: %+v", rules[0])
|
||||
}
|
||||
// The consequence, which is the reason this matters: removing web must not read as closing 443.
|
||||
nft := AsNftables(rules, nil, false, nil)
|
||||
nft := AsNftables(rules, nil, false, nil, nil)
|
||||
if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") {
|
||||
t.Fatalf("the rendered rule set does not name both sources:\n%s", nft)
|
||||
}
|
||||
@@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) {
|
||||
func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
// Naming the chain, not just the policy: the forward chain drops too, and an assertion on
|
||||
// "policy drop" alone passes while the input chain accepts everything. It did, once, here.
|
||||
if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") {
|
||||
@@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
|
||||
// `flush ruleset` would do the first and not the second: it empties every table on the machine,
|
||||
// including the ones the container runtime writes for its bridges.
|
||||
func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
|
||||
nft := AsNftables(nil, nil, false, nil)
|
||||
nft := AsNftables(nil, nil, false, nil, nil)
|
||||
if strings.Contains(nft, "flush ruleset") {
|
||||
t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft)
|
||||
}
|
||||
@@ -160,7 +160,7 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
|
||||
// So the chain exists and denies by default, and the runtime's own networks are allowed explicitly
|
||||
// — which is how the system being replaced has been doing it on these machines for months.
|
||||
func TestWhatIsForwardedIsGovernedToo(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "hook forward priority filter; policy drop") {
|
||||
t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft)
|
||||
}
|
||||
@@ -168,7 +168,7 @@ func TestWhatIsForwardedIsGovernedToo(t *testing.T) {
|
||||
|
||||
// And containers keep working, which is the whole reason the chain was left out before.
|
||||
func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil)
|
||||
for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} {
|
||||
if !strings.Contains(nft, "ip saddr "+network+" accept") {
|
||||
t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft)
|
||||
@@ -183,7 +183,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
|
||||
func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "ct original proto-dst 8080 accept") {
|
||||
t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft)
|
||||
}
|
||||
@@ -193,7 +193,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
|
||||
func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") {
|
||||
t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft)
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
|
||||
func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") {
|
||||
t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft)
|
||||
}
|
||||
@@ -213,7 +213,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
|
||||
func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), nil, false, nil)
|
||||
}}, nil), nil, false, nil, nil)
|
||||
if strings.Contains(nft, "dport 5432 accept") {
|
||||
t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft)
|
||||
}
|
||||
@@ -226,7 +226,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
|
||||
func TestAMachineScopedPortIsNotOpened(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
if strings.Contains(nft, "dport 6379 accept") {
|
||||
t.Fatalf("a port for this machine only was opened to the network:\n%s", nft)
|
||||
}
|
||||
@@ -268,7 +268,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) {
|
||||
func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil)
|
||||
}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") {
|
||||
t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft)
|
||||
}
|
||||
@@ -672,7 +672,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) {
|
||||
// loading the rules lives on conntrack until it drops, and then the machine is reached from a
|
||||
// rescue console (novox/hq issue 047).
|
||||
func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil)
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") {
|
||||
t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft)
|
||||
}
|
||||
@@ -685,7 +685,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
|
||||
// And from outside as well, on a machine that faces outward — because that is the way in when the
|
||||
// private network is the thing that broke.
|
||||
func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil)
|
||||
if !strings.Contains(nft, "\t\ttcp dport 22 accept") {
|
||||
t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft)
|
||||
}
|
||||
@@ -697,7 +697,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
|
||||
// to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody
|
||||
// adopts, reached over the network, closed by the act of adopting it.
|
||||
func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
|
||||
nft := AsNftables(nil, nil, false, nil)
|
||||
nft := AsNftables(nil, nil, false, nil, nil)
|
||||
if !strings.Contains(nft, "tcp dport 22 accept") {
|
||||
t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
|
||||
}
|
||||
|
||||
@@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) {
|
||||
rules := mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}},
|
||||
}}, nil)
|
||||
t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil))
|
||||
t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
-- The networks a machine routes for what it hosts, beyond the container runtime's own defaults.
|
||||
--
|
||||
-- novox/hq ADR 0137. The derived packet filter denies forwarding by default and then allows the
|
||||
-- container runtime's two default pools, named in the controller's code with a comment saying that
|
||||
-- a machine configured otherwise "needs this to say so" — and no way to say it. So the filter was
|
||||
-- correct only on a machine whose runtime used the defaults, and silently wrong on any other.
|
||||
--
|
||||
-- Measured on 2026-09-28: flipping a workstation to the derived filter cut egress for five of its
|
||||
-- container networks and for every network its test beds create, because those are allocated from
|
||||
-- ranges the two defaults do not cover. Nothing reported a fault; the containers simply could not
|
||||
-- reach anything.
|
||||
--
|
||||
-- A node-level fact, beside the node's public domain and for the same reason: it is a property of
|
||||
-- the machine, not of whichever module happens to load the filter today. Swapping that module must
|
||||
-- not lose it.
|
||||
--
|
||||
-- Null for a machine that routes nothing but the runtime's defaults, which is the ordinary case and
|
||||
-- what every machine held before this column existed.
|
||||
alter table node add column routed_networks jsonb;
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -518,6 +519,69 @@ func (i *Inventory) PublicDomainOf(ctx context.Context, name string) (string, er
|
||||
return *domain, nil
|
||||
}
|
||||
|
||||
// SetRoutedNetworks records the networks this machine routes for what it hosts, beyond the
|
||||
// container runtime's own default pools.
|
||||
//
|
||||
// A node-level fact (novox/hq ADR 0137), beside the node's public domain: the machine routes them,
|
||||
// not whichever module loads the filter, so swapping that module must not lose them. Added to the
|
||||
// runtime's defaults rather than replacing them, so a machine that says one range does not lose the
|
||||
// ranges its containers were already using. An empty list clears it.
|
||||
//
|
||||
// Each entry is checked as a CIDR here rather than at render time: an address that does not parse
|
||||
// becomes a line nftables refuses, and a refused ruleset is a machine that filters nothing while
|
||||
// its service reports a configuration fault.
|
||||
func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks []string) error {
|
||||
node, err := i.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var kept []string
|
||||
for _, n := range networks {
|
||||
n = strings.TrimSpace(n)
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
if _, _, err := net.ParseCIDR(n); err != nil {
|
||||
return fmt.Errorf("%q is not a network in CIDR form (10.0.0.0/8, 192.168.0.0/16): %w",
|
||||
n, err)
|
||||
}
|
||||
kept = append(kept, n)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set routed_networks = null where id = $1`, node.ID)
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(kept)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set routed_networks = $2 where id = $1`, node.ID, string(body))
|
||||
return err
|
||||
}
|
||||
|
||||
// RoutedNetworksOf is the networks a machine routes for what it hosts, empty when it has named none.
|
||||
func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string, error) {
|
||||
var body []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select routed_networks from node where name = $1`, name).Scan(&body)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var networks []string
|
||||
if err := json.Unmarshal(body, &networks); err != nil {
|
||||
return nil, fmt.Errorf("the networks recorded for %s are not a list: %w", name, err)
|
||||
}
|
||||
return networks, nil
|
||||
}
|
||||
|
||||
// RecordOverlayKey keeps the public half a node generated.
|
||||
func (i *Inventory) RecordOverlayKey(ctx context.Context, node, key string) error {
|
||||
if strings.TrimSpace(key) == "" {
|
||||
|
||||
Reference in New Issue
Block a user