Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76ac3c99bd | ||
|
|
fe5988c536 | ||
|
|
ed5d467d90 |
@@ -7,7 +7,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -310,7 +309,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", err
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != "", routed: with.Routed}
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
||||
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||
preview += "\n\n preview " + saw
|
||||
if !yes {
|
||||
@@ -415,16 +414,17 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
||||
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
||||
"declares it\n")
|
||||
// What it routes, said rather than left to the sentence above (novox/hq ADR 0137). The filter
|
||||
// forwards the container runtime's own default pools without being told; anything else is this
|
||||
// list, and a machine whose guests live outside those pools and names none of them loses their
|
||||
// egress at the flip, silently, which is how this was found.
|
||||
if len(derived.routed) > 0 {
|
||||
b.WriteString(fmt.Sprintf(" it routes: %s — kept forwarded, and their guests keep "+
|
||||
"address and name service\n", strings.Join(derived.routed, ", ")))
|
||||
// Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR
|
||||
// 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which
|
||||
// is what a reader most wants to know, because the previous shape of this filter cut a machine's
|
||||
// guests off at the flip without saying so, and that is how this was found.
|
||||
if len(derived.outwardLinks) > 0 {
|
||||
b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+
|
||||
"— everything its own guests send keeps working\n",
|
||||
strings.Join(derived.outwardLinks, ", ")))
|
||||
} else {
|
||||
b.WriteString(" it routes: nothing said, so only the container runtime's own default " +
|
||||
"pools are forwarded — `node networks <node> <cidr>...` if its guests live elsewhere\n")
|
||||
b.WriteString(" it has reported no link facing outside, so no filter can be composed " +
|
||||
"for it — the flip is refused until it reports one\n")
|
||||
}
|
||||
|
||||
isTaken := map[string]bool{}
|
||||
@@ -485,9 +485,10 @@ type derivedFilter struct {
|
||||
// mesh is every address on the private network; outward says the machine faces outside.
|
||||
mesh []string
|
||||
outward bool
|
||||
// routed is the networks this machine says it routes for what it hosts (novox/hq ADR 0137):
|
||||
// their guests keep address and name service, and what they send onward keeps being forwarded.
|
||||
routed []string
|
||||
// outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140).
|
||||
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
||||
// own guest and is not filtered.
|
||||
outwardLinks []string
|
||||
}
|
||||
|
||||
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||
@@ -513,12 +514,11 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
return "stays open — the mesh's own, from anywhere"
|
||||
}
|
||||
}
|
||||
// A guest on a network this machine routes asks it for an address and for names, and those two
|
||||
// arrive here (novox/hq ADR 0137). Matched on the listener's own address: a resolver bound to a
|
||||
// bridge in one of those networks is the one its guests ask.
|
||||
if within(r.Address, d.routed) &&
|
||||
((r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53)) {
|
||||
return "stays open — address and name service for a network this machine routes"
|
||||
// This machine's own guests ask it for an address and for names, and those two arrive here
|
||||
// (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an
|
||||
// outward link keeps answering them.
|
||||
if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) {
|
||||
return "stays open — this machine's own guests asking it for an address and for names"
|
||||
}
|
||||
for _, rule := range d.rules {
|
||||
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
||||
@@ -542,24 +542,6 @@ func (d derivedFilter) fate(r inventory.Reach) string {
|
||||
return "WILL CLOSE — no module assigned here declares it"
|
||||
}
|
||||
|
||||
// within says whether an address the machine reported sits inside one of the networks it routes.
|
||||
func within(address string, networks []string) bool {
|
||||
ip := net.ParseIP(strings.Trim(address, "[]"))
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
for _, n := range networks {
|
||||
_, block, err := net.ParseCIDR(n)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if block.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// countReachable is how much of a node's account of itself names something off the machine.
|
||||
// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
|
||||
// so a report of loopback alone says nothing about what the filter would close.
|
||||
|
||||
@@ -21,8 +21,7 @@ import (
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, " + publicDomainUsage +
|
||||
", or " + networksUsage)
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -68,10 +67,16 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
return publicDomain(ctx, inv, args[1:])
|
||||
|
||||
case "networks":
|
||||
// The networks this machine routes for what it hosts (novox/hq ADR 0137): what the derived
|
||||
// filter must keep forwarding, beyond the container runtime's own default pools which it
|
||||
// allows without being told. Reports with no argument, for the same reason the domain does.
|
||||
return nodeNetworks(ctx, inv, args[1:])
|
||||
// Removed by novox/hq ADR 0140, which superseded the record that added it. The filter no
|
||||
// longer names any network: it constrains what arrives from outside the machine and says
|
||||
// nothing about what did not, so there is no list to keep. Answered rather than met with
|
||||
// "unknown command", because this was the documented way to stop a flip cutting a machine's
|
||||
// containers off and somebody will reasonably still type it.
|
||||
return errors.New("`node networks` is gone (novox/hq ADR 0140). The filter constrains what " +
|
||||
"arrives from outside this machine and says nothing about traffic that did not, so no " +
|
||||
"network is named anywhere and nothing needs to be said to keep a machine's own " +
|
||||
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||
"`node show <name>`")
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
@@ -151,67 +156,6 @@ func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []st
|
||||
return nil
|
||||
}
|
||||
|
||||
const networksUsage = "node networks <name> — what it routes now; " +
|
||||
"<name> <cidr>... to set them; <name> --clear to route only the container runtime's own"
|
||||
|
||||
// nodeNetworks reads, sets or clears the networks a machine routes for what it hosts.
|
||||
//
|
||||
// The same three forms as the domain above, and the read-shaped one reports rather than clearing,
|
||||
// for the same reason: this list is what keeps a machine's guests reaching anything, and losing it
|
||||
// by asking a question is not a mistake anybody can see afterwards.
|
||||
func nodeNetworks(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node networks", flag.ContinueOnError)
|
||||
clear := set.Bool("clear", false,
|
||||
"route only the container runtime's own default pools, as a machine that has said nothing does")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 {
|
||||
return errors.New(networksUsage)
|
||||
}
|
||||
node := positionals[0]
|
||||
|
||||
switch {
|
||||
case *clear && len(positionals) > 1:
|
||||
return fmt.Errorf("give %s networks or --clear, not both: %q and --clear say opposite "+
|
||||
"things and the mesh will not choose between them", node, strings.Join(positionals[1:], " "))
|
||||
|
||||
case *clear:
|
||||
if err := inv.SetRoutedNetworks(ctx, node, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
|
||||
fmt.Printf(" run `push %s` to send its filter\n", node)
|
||||
return nil
|
||||
|
||||
case len(positionals) > 1:
|
||||
if err := inv.SetRoutedNetworks(ctx, node, positionals[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s routes %s\n", node, strings.Join(positionals[1:], ", "))
|
||||
fmt.Printf(" its filter forwards them, and their guests keep address and name service\n")
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
|
||||
default:
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
networks, err := inv.RoutedNetworksOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(networks) == 0 {
|
||||
fmt.Printf("%s routes only the container runtime's own default pools\n", node)
|
||||
fmt.Printf(" `node networks %s <cidr>...` if its guests live elsewhere\n", node)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s routes %s\n", node, strings.Join(networks, ", "))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
|
||||
@@ -640,9 +640,9 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// The networks this machine routes for what it hosts, which the derived filter must forward for
|
||||
// (novox/hq ADR 0137).
|
||||
routed, err := inv.RoutedNetworksOf(ctx, node)
|
||||
// Which of this machine's links face outside, which is what the derived filter is written
|
||||
// around (novox/hq ADR 0140). Reported by the machine, never set.
|
||||
outwardLinks, err := inv.OutwardLinksOf(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -651,8 +651,8 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, Routed: routed,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
}, record, nil
|
||||
|
||||
@@ -59,7 +59,11 @@ func anchorRendering(adopted bool) Rendering {
|
||||
Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
|
||||
Mesh: []string{"10.42.0.1"},
|
||||
Foundation: []int{5671},
|
||||
Adopted: adopted,
|
||||
// What the machine reported faces outside, which every rule in the filter is written
|
||||
// around (novox/hq ADR 0140).
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
Adopted: adopted,
|
||||
// Genesis takes the foundation's modules.
|
||||
Taken: map[string]bool{"postgres": true, "lavinmq": true},
|
||||
}
|
||||
@@ -575,11 +579,13 @@ func TestAGivenMachineSideReachesTheFilterTheOpeningAndTheConsumer(t *testing.T)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
with := Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)},
|
||||
Given: map[string]map[int]int{"forge": given},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
Taken: map[string]bool{"forge": true},
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)},
|
||||
Given: map[string]map[int]int{"forge": given},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
Taken: map[string]bool{"forge": true},
|
||||
}
|
||||
|
||||
// What the runtime is handed: the machine's own port on the outside, the container's within.
|
||||
@@ -660,9 +666,11 @@ func TestALongFormPortIsOpenedWhereTheManifestPublishesIt(t *testing.T) {
|
||||
forge := aForge()
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
composed, err := r.Compose(Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -124,10 +124,15 @@ type Rendering struct {
|
||||
// nothing on this node keeps them, or the mesh has no operator key.
|
||||
Kept *KeptExport
|
||||
|
||||
// Routed is the networks this machine routes for what it hosts, beyond the container runtime's
|
||||
// own default pools, which the filter allows without being told (novox/hq ADR 0137). A node-level
|
||||
// fact: the machine routes them, and the module that loads the filter may be replaced.
|
||||
Routed []string
|
||||
// OutwardLinks is the links this machine reported as facing outside it, which the filter is
|
||||
// written around (novox/hq ADR 0140). Empty means the machine has not said, and the mesh
|
||||
// composes no filter for it rather than writing a rule around a link with no name.
|
||||
OutwardLinks []string
|
||||
|
||||
// TunnelInterface is the interface the mesh's private network runs on, named here rather than
|
||||
// imported because the overlay package rests on this one. Traffic arriving on it is the mesh's,
|
||||
// not this machine's own guest, so the filter admits it only by a rule.
|
||||
TunnelInterface string
|
||||
|
||||
// Foundation is the ports the mesh itself needs reachable on every machine, which no module
|
||||
// declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker
|
||||
@@ -350,7 +355,21 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation, with.Routed)
|
||||
// **A machine that has not said which links face outside is sent no filter** (novox/hq ADR
|
||||
// 0140). The whole chain is written around those links: with none, the rule that lets this
|
||||
// machine's own guests keep working would name an empty set, which nftables refuses, and a rule
|
||||
// set that does not load is a machine filtering nothing while its unit reports success. Refused
|
||||
// here, where a person reads it, rather than on the machine — and the machine keeps the filter
|
||||
// it already has.
|
||||
if filters := r.filtersHere(); filters != "" && len(with.OutwardLinks) == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"%s cannot be sent a filter: it has not reported which of its links face outside, and "+
|
||||
"every rule in the chain is written around them. It reports that on each apply; "+
|
||||
"`node show %s` says whether it has. Until then %s is not sent, and the machine "+
|
||||
"keeps the filter it has", r.Node, r.Node, filters)
|
||||
}
|
||||
filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Foundation,
|
||||
with.OutwardLinks, with.TunnelInterface)
|
||||
|
||||
var out []map[string]any
|
||||
for _, m := range r.Modules {
|
||||
@@ -857,6 +876,17 @@ func mapping(written string) (outer, inner int, address string, ok bool) {
|
||||
return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true
|
||||
}
|
||||
|
||||
// filtersHere is the module on this node that loads the machine's packet filter, or empty when none
|
||||
// does. Named rather than counted: a refusal that says which module is one step from acted on.
|
||||
func (r Resolution) filtersHere() string {
|
||||
for _, m := range r.Modules {
|
||||
if m.Filtering != nil {
|
||||
return m.Module
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Rules is the rule set this node's filter is derived from: every module's listens, what was
|
||||
// computed for this machine, and each module's per-node exposure. The same answer whether the node
|
||||
// is adopted or converged — the one loads it as a filter, the other declares it as openings.
|
||||
|
||||
@@ -230,7 +230,23 @@ const SSHPort = 22
|
||||
// It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can
|
||||
// repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing
|
||||
// any module asks for, and neither may be derived away.
|
||||
func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, routed []string) string {
|
||||
func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
outwardLinks []string, tunnel string) string {
|
||||
// The links that are not this machine's own: the ones facing outside, and the mesh's tunnel.
|
||||
// Traffic arriving on any of them is admitted only by a rule below; traffic arriving anywhere
|
||||
// else is this machine's own guest and is not something the mesh has a position on.
|
||||
//
|
||||
// The tunnel is named here deliberately. Treating it as "not outside" would make a port nothing
|
||||
// declares reachable from every machine in the mesh, which is the derivation abandoned.
|
||||
quoted := make([]string, 0, len(outwardLinks)+1)
|
||||
for _, link := range outwardLinks {
|
||||
quoted = append(quoted, fmt.Sprintf("%q", link))
|
||||
}
|
||||
if tunnel != "" {
|
||||
quoted = append(quoted, fmt.Sprintf("%q", tunnel))
|
||||
}
|
||||
inward := strings.Join(quoted, ", ")
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("# Computed by the mesh from what is assigned to this node.\n")
|
||||
b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n")
|
||||
@@ -252,19 +268,20 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou
|
||||
b.WriteString("\t\ticmp type echo-request accept\n")
|
||||
b.WriteString("\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n")
|
||||
|
||||
// **What a machine it routes for must be able to ask it** (novox/hq ADR 0137). A guest on one of
|
||||
// these networks gets its address and its names from this machine, over the bridge it is on, and
|
||||
// those two questions arrive at the input chain like any other. Denied, the guest never gets an
|
||||
// address and never resolves a name — which is not "a closed port" but a network that does not
|
||||
// work at all, and it is this machine's own guest asking.
|
||||
// **What this machine's own guests must be able to ask it** (novox/hq ADR 0140). A guest gets
|
||||
// its address and its names from this machine, over the link it is on, and those two questions
|
||||
// arrive at the input chain like any other. Denied, the guest never gets an address and never
|
||||
// resolves a name — which is not "a closed port" but a network that does not work at all, and it
|
||||
// is this machine's own guest asking.
|
||||
//
|
||||
// Only these ports, and only for a network that was named: everything else a guest might want
|
||||
// from its host is a port somebody declares, like every other port on this machine.
|
||||
for _, network := range routed {
|
||||
family := saddrFamily(network)
|
||||
b.WriteString("\t\t# address and name service for a network this machine routes\n")
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s udp dport { 53, 67 } accept\n", family, network))
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s tcp dport 53 accept\n", family, network))
|
||||
// Asked for by the link it arrives on rather than by the address it comes from, for the reason
|
||||
// the forward chain below no longer names an address: a range describes one machine and goes
|
||||
// stale in silence. Anything arriving from outside, or over the tunnel, is not a guest of this
|
||||
// machine and asks through a port somebody declared, like everything else.
|
||||
if len(inward) > 0 {
|
||||
b.WriteString("\t\t# this machine's own guests asking it for an address and for names\n")
|
||||
b.WriteString(fmt.Sprintf("\t\tiifname != { %s } udp dport { 53, 67 } accept\n", inward))
|
||||
b.WriteString(fmt.Sprintf("\t\tiifname != { %s } tcp dport 53 accept\n", inward))
|
||||
}
|
||||
|
||||
// **ssh, always, and not because a module asked.**
|
||||
@@ -376,26 +393,36 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou
|
||||
// about the ports most worth protecting. Rehearsed on three machines: loading these rules
|
||||
// refused a port on the host and left a published container port reachable (novox/hq issue 047).
|
||||
//
|
||||
// The way through is the one the system being replaced already used: deny by default here, and
|
||||
// then explicitly allow the runtime's own networks, so containers keep working while everything
|
||||
// else has to be asked for.
|
||||
// **What it constrains is traffic arriving from OUTSIDE this machine, and nothing else**
|
||||
// (novox/hq ADR 0140).
|
||||
//
|
||||
// It used to deny everything here and then allow the machine's own containers back by naming
|
||||
// the address ranges they sit on — two ranges fixed in this file and the rest recorded per
|
||||
// machine. Every way of keeping that list correct failed. A constant describes one machine. A
|
||||
// recorded range goes stale in silence and cannot tell a network the mesh made from one a
|
||||
// predecessor left behind. Generating it from the modules would have put half this rule set on
|
||||
// the machine.
|
||||
//
|
||||
// The list should not exist, because the mesh has no position on a container reaching outward:
|
||||
// that is not a port opened to anybody. So traffic that did not arrive from outside is accepted
|
||||
// in one line, and what did arrive from outside is allowed only where a rule below admits it.
|
||||
//
|
||||
// The tunnel is not "not outside". Accepting everything off it would make a port nothing
|
||||
// declares reachable from any machine in the mesh, which is the derivation abandoned — so it is
|
||||
// named here beside the outward links, and traffic arriving on it meets the rules below like
|
||||
// anything else.
|
||||
b.WriteString("\tchain forward {\n")
|
||||
b.WriteString("\t\ttype filter hook forward priority filter; policy drop;\n")
|
||||
b.WriteString("\t\tct state established,related accept\n")
|
||||
b.WriteString("\t\tct state invalid drop\n")
|
||||
b.WriteString("\n")
|
||||
// What the container runtime created. Without these, denying by default stops every container
|
||||
// on the machine — which is exactly the failure the absent chain was avoiding, avoided properly.
|
||||
for _, network := range runtimeNetworks {
|
||||
b.WriteString(fmt.Sprintf("\t\t# %s\n", network.why))
|
||||
b.WriteString(fmt.Sprintf("\t\tip saddr %s accept\n", network.cidr))
|
||||
}
|
||||
// And what this machine says it routes beyond them (novox/hq ADR 0137). Added to the defaults
|
||||
// above, never replacing them: a machine that names one range has not stopped hosting whatever
|
||||
// was already on the runtime's own.
|
||||
for _, network := range routed {
|
||||
b.WriteString("\t\t# a network this machine routes for what it hosts\n")
|
||||
b.WriteString(fmt.Sprintf("\t\t%s saddr %s accept\n", saddrFamily(network), network))
|
||||
// Only when there is a link to name. An empty set is a line nftables refuses, and a rule set
|
||||
// that does not load is a machine filtering nothing while its unit reports success — so the
|
||||
// chain denies rather than renders nonsense. Composing a declaration for a machine that has
|
||||
// named none is refused upstream, so this is a floor and not a path anything travels.
|
||||
if inward != "" {
|
||||
b.WriteString("\t\t# this machine's own guests reaching outward: not a port opened to anybody\n")
|
||||
b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward))
|
||||
}
|
||||
|
||||
if len(rules) > 0 {
|
||||
@@ -452,28 +479,6 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int, rou
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// runtimeNetworks are the container runtime's own networks, which must keep working when the
|
||||
// forward chain denies by default.
|
||||
//
|
||||
// Taken from what the system being replaced allows, which has been carrying this machine's traffic
|
||||
// for months: the runtime's bridge range and the range its compose files are given. A machine whose
|
||||
// runtime is configured with something else needs this to say so — which is a thing the mesh cannot
|
||||
// derive and a reason this list is named here rather than computed.
|
||||
var runtimeNetworks = []struct{ cidr, why string }{
|
||||
{"172.16.0.0/12", "the container runtime's bridge networks"},
|
||||
{"192.168.128.0/17", "the networks its compose files are given"},
|
||||
}
|
||||
|
||||
// saddrFamily is the match a network's family is written with: `ip saddr` or `ip6 saddr`. One match
|
||||
// for both families is a syntax error, and a ruleset that does not load is a machine filtering
|
||||
// nothing while its service reports a fault — the same reason byFamily below exists.
|
||||
func saddrFamily(network string) string {
|
||||
if strings.Contains(network, ":") {
|
||||
return "ip6"
|
||||
}
|
||||
return "ip"
|
||||
}
|
||||
|
||||
// byFamily splits addresses into the two nftables understands separately.
|
||||
//
|
||||
// `ip saddr` and `ip6 saddr` are different matches, and one set holding both families is a syntax
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
|
||||
// A machine on the private network, with one ordinary module rule, and nothing that mentions
|
||||
// the broker — which is every machine.
|
||||
rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}}
|
||||
out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil)
|
||||
out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}, nil, "mesh0")
|
||||
|
||||
if !strings.Contains(out, "tcp dport 5671 accept") {
|
||||
t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out)
|
||||
@@ -48,7 +48,7 @@ func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
|
||||
// And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or
|
||||
// a panic. A control plane in that state cannot issue tokens either, which is where it surfaces.
|
||||
func TestNoBrokerMeansNoFoundationRuleRatherThanNoRuleset(t *testing.T) {
|
||||
out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil)
|
||||
out := AsNftables(nil, []string{"10.42.0.1"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(out, "table inet mesh") {
|
||||
t.Fatalf("no ruleset at all:\n%s", out)
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A machine's own guests keep working when the filter it is given denies by default.
|
||||
//
|
||||
// **The forward chain allowed the container runtime's two default pools and nothing else** — named
|
||||
// in this package's code with a comment saying a machine configured otherwise "needs this to say
|
||||
// so", and no way to say it (novox/hq ADR 0137). Measured on a workstation on 2026-09-28: the flip
|
||||
// to the derived filter cut egress for five of its container networks, allocated from ranges those
|
||||
// two defaults do not cover, and for every network its test beds create. Nothing reported a fault.
|
||||
// The guests simply could not reach anything, and the machine went on saying it had applied what it
|
||||
// was told.
|
||||
func TestWhatAMachineSaysItRoutesKeepsBeingForwarded(t *testing.T) {
|
||||
const beds = "10.0.0.0/8"
|
||||
|
||||
ruleset := AsNftables(nil, []string{"10.10.0.1"}, false, nil, []string{beds})
|
||||
|
||||
forward := chainOf(t, ruleset, "forward")
|
||||
if !strings.Contains(forward, "ip saddr "+beds+" accept") {
|
||||
t.Errorf("the forward chain does not accept what the machine says it routes (%s):\n%s",
|
||||
beds, forward)
|
||||
}
|
||||
// The defaults stay. A machine that names one range has not stopped hosting whatever was
|
||||
// already on the runtime's own pools, and losing those would trade one silent breakage for
|
||||
// another.
|
||||
for _, network := range runtimeNetworks {
|
||||
if !strings.Contains(forward, "ip saddr "+network.cidr+" accept") {
|
||||
t.Errorf("naming a network dropped the runtime's own %s:\n%s", network.cidr, forward)
|
||||
}
|
||||
}
|
||||
|
||||
// And its guests can still ask this machine the two questions that make a network usable at
|
||||
// all: what is my address, and what is that name.
|
||||
input := chainOf(t, ruleset, "input")
|
||||
for _, want := range []string{
|
||||
"ip saddr " + beds + " udp dport { 53, 67 } accept",
|
||||
"ip saddr " + beds + " tcp dport 53 accept",
|
||||
} {
|
||||
if !strings.Contains(input, want) {
|
||||
t.Errorf("the input chain is missing %q, so a guest on %s gets no address and "+
|
||||
"resolves no name:\n%s", want, beds, input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that says nothing is filtered exactly as it was before this existed.
|
||||
//
|
||||
// The change has to be additive on every machine already converged: novox has been carrying this
|
||||
// mesh's public services behind the derived filter for weeks, and a new line in its ruleset is a
|
||||
// change to a production firewall nobody asked for.
|
||||
func TestAMachineThatNamesNoNetworksIsFilteredAsBefore(t *testing.T) {
|
||||
rules := []Rule{{Port: 443, Protocol: "tcp", From: FromEverywhere, Because: []string{"proxy"}}}
|
||||
|
||||
said := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, nil)
|
||||
quiet := AsNftables(rules, []string{"10.10.0.1"}, true, []int{4222}, []string{})
|
||||
|
||||
if said != quiet {
|
||||
t.Errorf("nil and empty render differently:\n%s\n---\n%s", said, quiet)
|
||||
}
|
||||
if strings.Contains(said, "a network this machine routes") {
|
||||
t.Errorf("a machine that named nothing carries a line about what it routes:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The two families are matched differently, and one set holding both is a syntax error — a ruleset
|
||||
// that does not load is a machine filtering nothing while its unit reports a fault.
|
||||
func TestARoutedNetworkIsMatchedInItsOwnFamily(t *testing.T) {
|
||||
ruleset := AsNftables(nil, nil, false, nil, []string{"10.0.0.0/8", "fd00::/8"})
|
||||
|
||||
if !strings.Contains(ruleset, "ip saddr 10.0.0.0/8 accept") {
|
||||
t.Errorf("the v4 network is not matched as ip saddr:\n%s", ruleset)
|
||||
}
|
||||
if !strings.Contains(ruleset, "ip6 saddr fd00::/8 accept") {
|
||||
t.Errorf("the v6 network is not matched as ip6 saddr:\n%s", ruleset)
|
||||
}
|
||||
if strings.Contains(ruleset, "ip saddr fd00::/8") {
|
||||
t.Errorf("a v6 network is matched as ip saddr, which nftables refuses:\n%s", ruleset)
|
||||
}
|
||||
}
|
||||
|
||||
// chainOf is one chain's body, so a test about the forward chain cannot pass on a line in the input
|
||||
// chain that happens to look the same.
|
||||
func chainOf(t *testing.T, ruleset, name string) string {
|
||||
t.Helper()
|
||||
start := strings.Index(ruleset, "chain "+name+" {")
|
||||
if start < 0 {
|
||||
t.Fatalf("no chain %q in:\n%s", name, ruleset)
|
||||
}
|
||||
rest := ruleset[start:]
|
||||
end := strings.Index(rest, "\n\t}")
|
||||
if end < 0 {
|
||||
t.Fatalf("chain %q does not end:\n%s", name, rest)
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) {
|
||||
t.Fatalf("a module that wanted this port open is not named: %+v", rules[0])
|
||||
}
|
||||
// The consequence, which is the reason this matters: removing web must not read as closing 443.
|
||||
nft := AsNftables(rules, nil, false, nil, nil)
|
||||
nft := AsNftables(rules, nil, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") {
|
||||
t.Fatalf("the rendered rule set does not name both sources:\n%s", nft)
|
||||
}
|
||||
@@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) {
|
||||
func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0")
|
||||
// Naming the chain, not just the policy: the forward chain drops too, and an assertion on
|
||||
// "policy drop" alone passes while the input chain accepts everything. It did, once, here.
|
||||
if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") {
|
||||
@@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
|
||||
// `flush ruleset` would do the first and not the second: it empties every table on the machine,
|
||||
// including the ones the container runtime writes for its bridges.
|
||||
func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
|
||||
nft := AsNftables(nil, nil, false, nil, nil)
|
||||
nft := AsNftables(nil, nil, false, nil, nil, "mesh0")
|
||||
if strings.Contains(nft, "flush ruleset") {
|
||||
t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft)
|
||||
}
|
||||
@@ -160,22 +160,122 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
|
||||
// So the chain exists and denies by default, and the runtime's own networks are allowed explicitly
|
||||
// — which is how the system being replaced has been doing it on these machines for months.
|
||||
func TestWhatIsForwardedIsGovernedToo(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "hook forward priority filter; policy drop") {
|
||||
t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// And containers keep working, which is the whole reason the chain was left out before.
|
||||
func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil)
|
||||
for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} {
|
||||
if !strings.Contains(nft, "ip saddr "+network+" accept") {
|
||||
t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft)
|
||||
// And this machine's own guests keep working, which is the whole reason the chain was left out
|
||||
// before — by not being mentioned (novox/hq ADR 0140).
|
||||
//
|
||||
// It used to be done by naming the address ranges they sit on: two fixed here and the rest recorded
|
||||
// per machine. That list broke a workstation's containers at a flip and could not be made correct,
|
||||
// because a range describes one machine and cannot tell a network the mesh made from one a
|
||||
// predecessor left behind. What replaced it is a single line about the links traffic arrives on.
|
||||
func TestThisMachinesOwnGuestsKeepWorkingWithoutBeingNamed(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0")
|
||||
if !strings.Contains(nft, `iifname != { "eth0", "mesh0" } accept`) {
|
||||
t.Fatalf("what did not arrive from outside is not accepted, so this machine's own guests "+
|
||||
"reach nothing:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// No address of a machine's own networks appears anywhere in a rendered filter.
|
||||
//
|
||||
// This is the assertion that fails against the previous behaviour, and it is why it is written on
|
||||
// the text rather than on an outcome: the two ranges were a constant in this file, so nothing but
|
||||
// reading the output catches one creeping back in.
|
||||
func TestNoNetworkOfTheMachinesOwnIsNamed(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0")
|
||||
for _, gone := range []string{"172.16.0.0/12", "192.168.128.0/17", "saddr 192.168", "saddr 172."} {
|
||||
if strings.Contains(nft, gone) {
|
||||
t.Fatalf("%q is named, and a range describes one machine and goes stale in silence:\n%s",
|
||||
gone, nft)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The tunnel is constrained, not treated as inside.**
|
||||
//
|
||||
// Accepting everything arriving over the private network would make a port nothing declares
|
||||
// reachable from every machine in the mesh — the derivation abandoned, and a rule that reads as a
|
||||
// restriction while restricting nothing. So the tunnel is named beside the outward links, and
|
||||
// traffic arriving on it meets the declared rules like anything else.
|
||||
func TestTheTunnelIsConstrainedLikeAnOutwardLink(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0")
|
||||
line := `iifname != { "eth0", "mesh0" } accept`
|
||||
if !strings.Contains(nft, line) {
|
||||
t.Fatalf("the tunnel is not constrained, so an undeclared port is reachable from any "+
|
||||
"machine in the mesh:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine with two links facing outside has both constrained. Asserted on the one line, because a
|
||||
// rule covering one and not the other would leave a machine filtering half of what reaches it.
|
||||
func TestEveryOutwardLinkIsConstrained(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0", "wlan0"}, "mesh0")
|
||||
if !strings.Contains(nft, `iifname != { "eth0", "wlan0", "mesh0" } accept`) {
|
||||
t.Fatalf("not every outward link is constrained:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// A guest asks its host for an address and for names, and those two arrive at the input chain. Asked
|
||||
// for by the link they arrive on, so a resolver bound anywhere but an outward link keeps answering.
|
||||
func TestGuestsMayAskTheirHostForAnAddressAndNames(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, []string{"eth0"}, "mesh0")
|
||||
for _, want := range []string{
|
||||
`iifname != { "eth0", "mesh0" } udp dport { 53, 67 } accept`,
|
||||
`iifname != { "eth0", "mesh0" } tcp dport 53 accept`,
|
||||
} {
|
||||
if !strings.Contains(nft, want) {
|
||||
t.Fatalf("a guest cannot ask its host for an address or a name, which is not a closed "+
|
||||
"port but a network that does not work:\n%s", nft)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// With no link named at all the chain denies rather than rendering an empty set, which nftables
|
||||
// refuses — and a rule set that does not load is a machine filtering nothing while its unit reports
|
||||
// success. Composing a declaration for such a machine is refused upstream; this is the floor.
|
||||
func TestNoLinkNamedRendersNoCatchAllRatherThanAnEmptySet(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil, nil, "")
|
||||
if strings.Contains(nft, "{ }") || strings.Contains(nft, "iifname != {}") {
|
||||
t.Fatalf("an empty set is rendered, which nftables refuses:\n%s", nft)
|
||||
}
|
||||
if !strings.Contains(nft, "hook forward priority filter; policy drop") {
|
||||
t.Fatalf("the forward chain does not deny:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that has not said which links face outside is sent no filter, and the refusal names the
|
||||
// module that would have loaded it so the reader knows what is being withheld.
|
||||
func TestAMachineThatNamedNoOutwardLinkIsSentNoFilter(t *testing.T) {
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{
|
||||
{Module: "nftables", Filtering: &Filtering{Into: "/etc/mesh/filter.nft"}},
|
||||
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
|
||||
}}
|
||||
_, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}, TunnelInterface: "mesh0"})
|
||||
if err == nil {
|
||||
t.Fatal("a machine that named no outward link was sent a filter written around none")
|
||||
}
|
||||
for _, want := range []string{"anchor", "nftables", "face outside"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And a machine that names none but loads no filter is not refused: there is nothing to write.
|
||||
func TestAMachineWithNoFilterModuleIsNotRefused(t *testing.T) {
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{
|
||||
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
|
||||
}}
|
||||
if _, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}}); err != nil {
|
||||
t.Fatalf("a machine that loads no filter was refused one: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A published port is matched by what the client asked for, not by where the packet ends up.
|
||||
//
|
||||
// The runtime rewrites the destination before this chain sees it, so a rule naming the published
|
||||
@@ -183,7 +283,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
|
||||
func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "ct original proto-dst 8080 accept") {
|
||||
t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft)
|
||||
}
|
||||
@@ -193,7 +293,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
|
||||
func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") {
|
||||
t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft)
|
||||
}
|
||||
@@ -203,7 +303,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
|
||||
func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") {
|
||||
t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft)
|
||||
}
|
||||
@@ -213,7 +313,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
|
||||
func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), nil, false, nil, nil)
|
||||
}}, nil), nil, false, nil, nil, "mesh0")
|
||||
if strings.Contains(nft, "dport 5432 accept") {
|
||||
t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft)
|
||||
}
|
||||
@@ -226,7 +326,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
|
||||
func TestAMachineScopedPortIsNotOpened(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}},
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2"}, false, nil, nil, "mesh0")
|
||||
if strings.Contains(nft, "dport 6379 accept") {
|
||||
t.Fatalf("a port for this machine only was opened to the network:\n%s", nft)
|
||||
}
|
||||
@@ -238,7 +338,8 @@ func TestTheModuleAskingForTheRuleSetGetsEveryModulesPorts(t *testing.T) {
|
||||
{Module: "firewall", Filtering: &Filtering{Into: "/etc/mesh/filter.nft"}},
|
||||
{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
|
||||
}}
|
||||
out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}})
|
||||
out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"},
|
||||
OutwardLinks: []string{"eth0"}, TunnelInterface: "mesh0"})
|
||||
if err != nil {
|
||||
t.Fatalf("declaration: %v", err)
|
||||
}
|
||||
@@ -268,7 +369,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) {
|
||||
func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) {
|
||||
nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
|
||||
}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil)
|
||||
}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") {
|
||||
t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft)
|
||||
}
|
||||
@@ -296,7 +397,8 @@ func TestWhatTheMeshComputesIsAppliedBeforeWhatTheModuleDeclared(t *testing.T) {
|
||||
"restart-on": []any{"filtering"}},
|
||||
},
|
||||
}}}
|
||||
out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"}})
|
||||
out, err := r.Declaration(Rendering{Mesh: []string{"198.51.100.2"},
|
||||
OutwardLinks: []string{"eth0"}, TunnelInterface: "mesh0"})
|
||||
if err != nil {
|
||||
t.Fatalf("declaration: %v", err)
|
||||
}
|
||||
@@ -672,7 +774,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) {
|
||||
// loading the rules lives on conntrack until it drops, and then the machine is reached from a
|
||||
// rescue console (novox/hq issue 047).
|
||||
func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") {
|
||||
t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft)
|
||||
}
|
||||
@@ -685,7 +787,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
|
||||
// And from outside as well, on a machine that faces outward — because that is the way in when the
|
||||
// private network is the thing that broke.
|
||||
func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil)
|
||||
nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "\t\ttcp dport 22 accept") {
|
||||
t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft)
|
||||
}
|
||||
@@ -697,7 +799,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
|
||||
// to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody
|
||||
// adopts, reached over the network, closed by the act of adopting it.
|
||||
func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
|
||||
nft := AsNftables(nil, nil, false, nil, nil)
|
||||
nft := AsNftables(nil, nil, false, nil, nil, "mesh0")
|
||||
if !strings.Contains(nft, "tcp dport 22 accept") {
|
||||
t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
|
||||
}
|
||||
|
||||
@@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) {
|
||||
rules := mustFilter(t, Resolution{Modules: []Manifest{
|
||||
{Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}},
|
||||
}}, nil)
|
||||
t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil))
|
||||
t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil, nil, "mesh0"))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Which of a machine's links face outside it, replacing the networks it was told to say it routes.
|
||||
--
|
||||
-- novox/hq ADR 0140, superseding 0137 and 0139. The derived filter blocked everything passing
|
||||
-- through a machine and then allowed the machine's own containers back by naming the address ranges
|
||||
-- they sit on: two ranges fixed in the controller's source, the rest recorded by 0043's column.
|
||||
--
|
||||
-- Every route to a correct list fails. A constant describes one machine. A recorded range goes stale
|
||||
-- in silence, and cannot tell a network the mesh made from one a predecessor left behind — measured
|
||||
-- on the control-node, where six ranges fall outside the constants and two of the six belong to
|
||||
-- services the mesh does not run. Generating the list from the modules put half the rule set on the
|
||||
-- machine.
|
||||
--
|
||||
-- The list should not exist, because the mesh has no position on a container reaching outward: that
|
||||
-- is not a port opened to anybody. The filter constrains what arrives from OUTSIDE the machine and
|
||||
-- says nothing about what did not, which needs one fact instead of a list — which links "outside"
|
||||
-- arrives on.
|
||||
--
|
||||
-- Reported by the machine on every apply, never recorded by hand, so it cannot go stale. Null for a
|
||||
-- machine that has not reported yet; the mesh composes no filter for such a machine and leaves the
|
||||
-- one it has, because a rule written around a link with no name is a rule set that does not load.
|
||||
alter table node add column outward_links jsonb;
|
||||
|
||||
-- What 0043 recorded is not migrated into it. The ranges answered a question that no longer exists,
|
||||
-- and every machine that named one keeps working without it: the traffic those ranges allowed is now
|
||||
-- allowed by not having arrived from outside.
|
||||
alter table node drop column routed_networks;
|
||||
+25
-34
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -519,37 +518,28 @@ func (i *Inventory) PublicDomainOf(ctx context.Context, name string) (string, er
|
||||
return *domain, nil
|
||||
}
|
||||
|
||||
// SetRoutedNetworks records the networks this machine routes for what it hosts, beyond the
|
||||
// container runtime's own default pools.
|
||||
// RecordOutwardLinks keeps the links a machine reported as facing outside it.
|
||||
//
|
||||
// A node-level fact (novox/hq ADR 0137), beside the node's public domain: the machine routes them,
|
||||
// not whichever module loads the filter, so swapping that module must not lose them. Added to the
|
||||
// runtime's defaults rather than replacing them, so a machine that says one range does not lose the
|
||||
// ranges its containers were already using. An empty list clears it.
|
||||
// A reported fact, not a setting (novox/hq ADR 0140). It replaces the networks a machine used to be
|
||||
// told to say it routes: the filter blocked everything passing through and then allowed the machine's
|
||||
// own containers back by naming their address ranges, and every way of keeping that list correct
|
||||
// failed — a constant describes one machine, and a recorded range goes stale in silence. The filter
|
||||
// now constrains what arrives from outside and says nothing about what did not, and the one thing it
|
||||
// needs is which links "outside" arrives on. The machine reads that from its own routing table on
|
||||
// every apply, so it cannot go stale and nobody types it.
|
||||
//
|
||||
// Each entry is checked as a CIDR here rather than at render time: an address that does not parse
|
||||
// becomes a line nftables refuses, and a refused ruleset is a machine that filters nothing while
|
||||
// its service reports a configuration fault.
|
||||
func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks []string) error {
|
||||
node, err := i.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// An empty list clears it, which is what a machine with no route off itself reports. The mesh then
|
||||
// composes no filter for that machine at all.
|
||||
func (i *Inventory) RecordOutwardLinks(ctx context.Context, id string, links []string) error {
|
||||
var kept []string
|
||||
for _, n := range networks {
|
||||
n = strings.TrimSpace(n)
|
||||
if n == "" {
|
||||
continue
|
||||
for _, name := range links {
|
||||
if name = strings.TrimSpace(name); name != "" {
|
||||
kept = append(kept, name)
|
||||
}
|
||||
if _, _, err := net.ParseCIDR(n); err != nil {
|
||||
return fmt.Errorf("%q is not a network in CIDR form (10.0.0.0/8, 192.168.0.0/16): %w",
|
||||
n, err)
|
||||
}
|
||||
kept = append(kept, n)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set routed_networks = null where id = $1`, node.ID)
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set outward_links = null where id = $1`, id)
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(kept)
|
||||
@@ -557,15 +547,16 @@ func (i *Inventory) SetRoutedNetworks(ctx context.Context, name string, networks
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set routed_networks = $2 where id = $1`, node.ID, string(body))
|
||||
`update node set outward_links = $2 where id = $1`, id, string(body))
|
||||
return err
|
||||
}
|
||||
|
||||
// RoutedNetworksOf is the networks a machine routes for what it hosts, empty when it has named none.
|
||||
func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string, error) {
|
||||
// OutwardLinksOf is the links a machine reported as facing outside it, empty when it has reported
|
||||
// none — which is a machine the mesh composes no filter for.
|
||||
func (i *Inventory) OutwardLinksOf(ctx context.Context, name string) ([]string, error) {
|
||||
var body []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select routed_networks from node where name = $1`, name).Scan(&body)
|
||||
`select outward_links from node where name = $1`, name).Scan(&body)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
@@ -575,11 +566,11 @@ func (i *Inventory) RoutedNetworksOf(ctx context.Context, name string) ([]string
|
||||
if len(body) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var networks []string
|
||||
if err := json.Unmarshal(body, &networks); err != nil {
|
||||
return nil, fmt.Errorf("the networks recorded for %s are not a list: %w", name, err)
|
||||
var links []string
|
||||
if err := json.Unmarshal(body, &links); err != nil {
|
||||
return nil, fmt.Errorf("the outward links recorded for %s are not a list: %w", name, err)
|
||||
}
|
||||
return networks, nil
|
||||
return links, nil
|
||||
}
|
||||
|
||||
// RecordOverlayKey keeps the public half a node generated.
|
||||
|
||||
@@ -301,6 +301,17 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
// Which of its links face outside (novox/hq ADR 0140), whenever it says so. Recorded on every
|
||||
// report that carries it, adopted or converged, because the filter the mesh composes is written
|
||||
// around it — and never cleared by a report that carries none, which is every bare word that the
|
||||
// node is there. A machine whose routing table it could not read reports nothing rather than
|
||||
// guessing, and keeps whatever it last said; a machine with genuinely no route off itself is one
|
||||
// the mesh composes no filter for at all.
|
||||
if len(report.Outward) > 0 {
|
||||
if err := e.Inventory.RecordOutwardLinks(ctx, node.ID, report.Outward); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
|
||||
if report.Tunnel != nil {
|
||||
if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{
|
||||
|
||||
@@ -170,6 +170,20 @@ type Report struct {
|
||||
// Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that
|
||||
// was never asked, which is every converged one.
|
||||
Firewall string `json:"firewall,omitempty"`
|
||||
|
||||
// Outward is the links on this machine that face outside it — the ones carrying a default route
|
||||
// (novox/hq ADR 0140). Every node reports it, adopted or converged, because the filter the mesh
|
||||
// composes for it is written around these and nothing else.
|
||||
//
|
||||
// **It replaces a list of addresses.** The filter used to block everything passing through the
|
||||
// machine and then allow the machine's own containers back by naming the ranges they sit on. A
|
||||
// range describes one machine and goes stale in silence; the link carrying the default route is
|
||||
// read afresh on every report and does not change when a module is added or removed.
|
||||
//
|
||||
// Empty means the machine has not said. The mesh composes no filter for such a machine and
|
||||
// leaves the one it has: a rule written around a link with no name is a rule set that does not
|
||||
// load, and that is a machine filtering nothing while its unit reports success.
|
||||
Outward []string `json:"outward,omitempty"`
|
||||
// Reachable is what can be reached on the machine now: every listening socket and every
|
||||
// published container port. Only an adopted node reports it; it is what converging previews.
|
||||
Reachable []Reach `json:"reachable,omitempty"`
|
||||
|
||||
@@ -346,7 +346,8 @@ func (s *Server) reported(ctx context.Context, m Control) {
|
||||
// whenever it arrives, which is the behaviour the mesh has had all along.
|
||||
func staleAgainst(report Report) string {
|
||||
if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 ||
|
||||
report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 {
|
||||
report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 ||
|
||||
len(report.Outward) > 0 {
|
||||
return ""
|
||||
}
|
||||
return report.Declared
|
||||
|
||||
Reference in New Issue
Block a user