diff --git a/cmd/mesh-control/plan.go b/cmd/mesh-control/plan.go index 260816a..a37e1a9 100644 --- a/cmd/mesh-control/plan.go +++ b/cmd/mesh-control/plan.go @@ -9,9 +9,12 @@ import ( "sort" "strings" + "github.com/novox/mesh-control/internal/broker" "github.com/novox/mesh-control/internal/catalogue" "github.com/novox/mesh-control/internal/inventory" "github.com/novox/mesh-control/internal/licences" + "net" + "strconv" ) // working out what one machine should be. @@ -447,9 +450,22 @@ func declarationWith(ctx context.Context, open *stores, node string, names[name] = at } + // The ports the mesh itself needs open, which no module declares. Read from the broker this + // control plane was told about rather than written down twice: the address a node is handed in + // its token and the port its machine must accept on are the same fact. + var substrate []int + if b, err := broker.FromEnvironment(); err == nil { + if _, port, err := net.SplitHostPort(b.Address); err == nil { + if n, err := strconv.Atoi(port); err == nil { + substrate = append(substrate, n) + } + } + } + return plan.Declaration(catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, - Certificate: certificate, Authority: authority, Mesh: private, Names: names}) + Certificate: certificate, Authority: authority, Mesh: private, Names: names, + Substrate: substrate}) } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index e91c3da..3b4a1c5 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -89,6 +89,12 @@ type Rendering struct { // a fact about the mesh, and resolution answers questions about one machine. Mesh []string + // Substrate is the ports the mesh itself needs reachable on every machine, which no module + // declares because the substrate is not a module (novox/hq 04-ISSUES/051 and 052). The broker + // is the one that matters: a machine dials it to enrol, and a firewall derived only from + // modules closes it. + Substrate []int + // Names is every machine's internal name and its address, for containers to be given. // // **A container does not inherit the machine's names**, so every internal name the mesh wrote @@ -207,7 +213,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { if err != nil { return nil, err } - filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "") + filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Substrate) var out []map[string]any for _, m := range r.Modules { diff --git a/internal/catalogue/filtering.go b/internal/catalogue/filtering.go index e9dbef7..ba1d6dc 100644 --- a/internal/catalogue/filtering.go +++ b/internal/catalogue/filtering.go @@ -216,7 +216,20 @@ const SSHPort = 22 // // `outward` says this machine is reachable from outside the mesh, which is the only thing that // decides whether ssh is answered there as well as on the private network. -func AsNftables(rules []Rule, mesh []string, outward bool) string { +// +// `substrate` is the ports the MESH ITSELF needs reachable, which no module declares. +// +// **Everything else in this file is derived from what modules say they listen on, and the +// substrate is not a module** (novox/hq 04-ISSUES/051). So the broker — the port every machine +// dials to enrol and to receive every declaration it is ever sent — was absent from the ruleset, +// and nothing noticed: a mesh of one never dials its own broker across the network. The first +// machine to join a firewalled anchor is refused by the packet filter during enrolment, before +// the mesh can report anything about it. +// +// It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can +// repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing +// any module asks for, and neither may be derived away. +func AsNftables(rules []Rule, mesh []string, outward bool, substrate []int) string { var b strings.Builder b.WriteString("# Computed by the mesh from what is assigned to this node.\n") b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n") @@ -279,6 +292,18 @@ func AsNftables(rules []Rule, mesh []string, outward bool) string { b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", SSHPort)) } + // The mesh's own ports, from anywhere. + // + // Not narrowed to the private network, because the machines that need this most are the ones + // not on it yet: a node enrols BEFORE it has an address here, over the ordinary network, and a + // rule allowing only the private network would close the door being knocked on. + if len(substrate) > 0 { + b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n") + for _, port := range substrate { + b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", port)) + } + } + if len(rules) > 0 { b.WriteString("\n") } diff --git a/internal/catalogue/filtering_substrate_test.go b/internal/catalogue/filtering_substrate_test.go new file mode 100644 index 0000000..7df5680 --- /dev/null +++ b/internal/catalogue/filtering_substrate_test.go @@ -0,0 +1,48 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The mesh's own ports survive a ruleset derived from modules that do not mention them. +// +// **The firewall is computed from what modules declare they listen on, and the substrate is not a +// module** (novox/hq 04-ISSUES/052). So the broker's port — the one every machine dials to enrol +// and to receive every declaration it is ever sent — was absent from every ruleset the mesh ever +// generated, and nothing caught it: a mesh of one never dials its own broker across the network, +// so the ruleset looks complete right up until a second machine tries to join and is refused by +// the packet filter, during enrolment, before the mesh can report anything about it. +func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) { + const brokerPort = 5671 + + // A machine on the private network, with one ordinary module rule, and nothing that mentions + // the broker — which is every machine. + rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}} + out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort}) + + if !strings.Contains(out, "tcp dport 5671 accept") { + t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out) + } + + // From anywhere, deliberately: a node enrols BEFORE it has an address on the private network, + // so a rule narrowed to that network would close the door being knocked on. + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "5671") && strings.Contains(line, "saddr") { + t.Fatalf("the broker's port is narrowed to the private network, which a machine that "+ + "has not yet enrolled is not on:\n%s", line) + } + } +} + +// And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or +// a panic. A control plane in that state cannot issue tokens either, which is where it surfaces. +func TestNoBrokerMeansNoSubstrateRuleRatherThanNoRuleset(t *testing.T) { + out := AsNftables(nil, []string{"10.42.0.1"}, false, nil) + if !strings.Contains(out, "table inet mesh") { + t.Fatalf("no ruleset at all:\n%s", out) + } + if strings.Contains(out, "never derived, never closed\n\t\ttcp dport") { + t.Fatalf("a substrate rule was written for a mesh with no broker:\n%s", out) + } +} diff --git a/internal/catalogue/filtering_test.go b/internal/catalogue/filtering_test.go index ddfd9fb..22a4e3f 100644 --- a/internal/catalogue/filtering_test.go +++ b/internal/catalogue/filtering_test.go @@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) { t.Fatalf("a module that wanted this port open is not named: %+v", rules[0]) } // The consequence, which is the reason this matters: removing web must not read as closing 443. - nft := AsNftables(rules, nil, false) + nft := AsNftables(rules, nil, false, nil) if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") { t.Fatalf("the rendered rule set does not name both sources:\n%s", nft) } @@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) { func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) // Naming the chain, not just the policy: the forward chain drops too, and an assertion on // "policy drop" alone passes while the input chain accepts everything. It did, once, here. if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") { @@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) { // `flush ruleset` would do the first and not the second: it empties every table on the machine, // including the ones the container runtime writes for its bridges. func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { - nft := AsNftables(nil, nil, false) + nft := AsNftables(nil, nil, false, nil) if strings.Contains(nft, "flush ruleset") { t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft) } @@ -160,7 +160,7 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) { // So the chain exists and denies by default, and the runtime's own networks are allowed explicitly // — which is how the system being replaced has been doing it on these machines for months. func TestWhatIsForwardedIsGovernedToo(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "hook forward priority filter; policy drop") { t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft) } @@ -168,7 +168,7 @@ func TestWhatIsForwardedIsGovernedToo(t *testing.T) { // And containers keep working, which is the whole reason the chain was left out before. func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, false) + nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil) for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} { if !strings.Contains(nft, "ip saddr "+network+" accept") { t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft) @@ -183,7 +183,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) { func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "ct original proto-dst 8080 accept") { t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft) } @@ -193,7 +193,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) { func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") { t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft) } @@ -203,7 +203,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) { func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false) + }}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") { t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft) } @@ -213,7 +213,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) { func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), nil, false) + }}, nil), nil, false, nil) if strings.Contains(nft, "dport 5432 accept") { t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft) } @@ -226,7 +226,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) { func TestAMachineScopedPortIsNotOpened(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}}, - }}, nil), []string{"198.51.100.2"}, false) + }}, nil), []string{"198.51.100.2"}, false, nil) if strings.Contains(nft, "dport 6379 accept") { t.Fatalf("a port for this machine only was opened to the network:\n%s", nft) } @@ -268,7 +268,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) { func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) { nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{ {Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}}, - }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false) + }}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") { t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft) } @@ -672,7 +672,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) { // loading the rules lives on conntrack until it drops, and then the machine is reached from a // rescue console (novox/hq issue 047). func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false) + nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil) if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") { t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft) } @@ -685,7 +685,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) { // And from outside as well, on a machine that faces outward — because that is the way in when the // private network is the thing that broke. func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { - nft := AsNftables(nil, []string{"198.51.100.2"}, true) + nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil) if !strings.Contains(nft, "\t\ttcp dport 22 accept") { t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft) } @@ -697,7 +697,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) { // to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody // adopts, reached over the network, closed by the act of adopting it. func TestSSHIsNeverLeftWithoutARule(t *testing.T) { - nft := AsNftables(nil, nil, false) + nft := AsNftables(nil, nil, false, nil) if !strings.Contains(nft, "tcp dport 22 accept") { t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft) } diff --git a/internal/catalogue/print_rehearsal_test.go b/internal/catalogue/print_rehearsal_test.go index 71e3dbb..65fbb58 100644 --- a/internal/catalogue/print_rehearsal_test.go +++ b/internal/catalogue/print_rehearsal_test.go @@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) { rules := mustFilter(t, Resolution{Modules: []Manifest{ {Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}}, }}, nil) - t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true)) + t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil)) }