cmd/mesh-control/plan.go

@@ -9,9 +9,12 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/novox/mesh-control/internal/broker"
 	"github.com/novox/mesh-control/internal/catalogue"
 	"github.com/novox/mesh-control/internal/inventory"
 	"github.com/novox/mesh-control/internal/licences"
+	"net"
+	"strconv"
 )
 
 // working out what one machine should be.
@@ -447,9 +450,22 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		names[name] = at
 	}
 
+	// The ports the mesh itself needs open, which no module declares. Read from the broker this
+	// control plane was told about rather than written down twice: the address a node is handed in
+	// its token and the port its machine must accept on are the same fact.
+	var substrate []int
+	if b, err := broker.FromEnvironment(); err == nil {
+		if _, port, err := net.SplitHostPort(b.Address); err == nil {
+			if n, err := strconv.Atoi(port); err == nil {
+				substrate = append(substrate, n)
+			}
+		}
+	}
+
 	return plan.Declaration(catalogue.Rendering{
 		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
-		Certificate: certificate, Authority: authority, Mesh: private, Names: names})
+		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
+		Substrate: substrate})
 }
 
 // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq

internal/catalogue/declaration.go

@@ -89,6 +89,12 @@ type Rendering struct {
 	// a fact about the mesh, and resolution answers questions about one machine.
 	Mesh []string
 
+	// Substrate is the ports the mesh itself needs reachable on every machine, which no module
+	// declares because the substrate is not a module (novox/hq 04-ISSUES/051 and 052). The broker
+	// is the one that matters: a machine dials it to enrol, and a firewall derived only from
+	// modules closes it.
+	Substrate []int
+
 	// Names is every machine's internal name and its address, for containers to be given.
 	//
 	// **A container does not inherit the machine's names**, so every internal name the mesh wrote
@@ -207,7 +213,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
 	if err != nil {
 		return nil, err
 	}
-	filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "")
+	filtering := AsNftables(rules, with.Mesh, r.PublicDomain != "", with.Substrate)
 
 	var out []map[string]any
 	for _, m := range r.Modules {

internal/catalogue/filtering.go

@@ -216,7 +216,20 @@ const SSHPort = 22
 //
 // `outward` says this machine is reachable from outside the mesh, which is the only thing that
 // decides whether ssh is answered there as well as on the private network.
-func AsNftables(rules []Rule, mesh []string, outward bool) string {
+//
+// `substrate` is the ports the MESH ITSELF needs reachable, which no module declares.
+//
+// **Everything else in this file is derived from what modules say they listen on, and the
+// substrate is not a module** (novox/hq 04-ISSUES/051). So the broker — the port every machine
+// dials to enrol and to receive every declaration it is ever sent — was absent from the ruleset,
+// and nothing noticed: a mesh of one never dials its own broker across the network. The first
+// machine to join a firewalled anchor is refused by the packet filter during enrolment, before
+// the mesh can report anything about it.
+//
+// It is a floor for the same reason ssh is. A machine nobody can reach is a machine nobody can
+// repair; a machine the mesh cannot reach is a machine the mesh cannot manage. Neither is a thing
+// any module asks for, and neither may be derived away.
+func AsNftables(rules []Rule, mesh []string, outward bool, substrate []int) string {
 	var b strings.Builder
 	b.WriteString("# Computed by the mesh from what is assigned to this node.\n")
 	b.WriteString("# Edits are lost on the next declaration; change a module's listens instead.\n\n")
@@ -279,6 +292,18 @@ func AsNftables(rules []Rule, mesh []string, outward bool) string {
 		b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", SSHPort))
 	}
 
+	// The mesh's own ports, from anywhere.
+	//
+	// Not narrowed to the private network, because the machines that need this most are the ones
+	// not on it yet: a node enrols BEFORE it has an address here, over the ordinary network, and a
+	// rule allowing only the private network would close the door being knocked on.
+	if len(substrate) > 0 {
+		b.WriteString("\n\t\t# the mesh's own — never derived, never closed\n")
+		for _, port := range substrate {
+			b.WriteString(fmt.Sprintf("\t\ttcp dport %d accept\n", port))
+		}
+	}
+
 	if len(rules) > 0 {
 		b.WriteString("\n")
 	}

internal/catalogue/filtering_substrate_test.go

@@ -0,0 +1,48 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// The mesh's own ports survive a ruleset derived from modules that do not mention them.
+//
+// **The firewall is computed from what modules declare they listen on, and the substrate is not a
+// module** (novox/hq 04-ISSUES/052). So the broker's port — the one every machine dials to enrol
+// and to receive every declaration it is ever sent — was absent from every ruleset the mesh ever
+// generated, and nothing caught it: a mesh of one never dials its own broker across the network,
+// so the ruleset looks complete right up until a second machine tries to join and is refused by
+// the packet filter, during enrolment, before the mesh can report anything about it.
+func TestTheBrokersPortIsOpenedThoughNoModuleDeclaresIt(t *testing.T) {
+	const brokerPort = 5671
+
+	// A machine on the private network, with one ordinary module rule, and nothing that mentions
+	// the broker — which is every machine.
+	rules := []Rule{{Port: 8080, From: FromMesh, Because: []string{"some-module"}}}
+	out := AsNftables(rules, []string{"10.42.0.1"}, false, []int{brokerPort})
+
+	if !strings.Contains(out, "tcp dport 5671 accept") {
+		t.Fatalf("the broker's port is not opened, so no machine could enrol:\n%s", out)
+	}
+
+	// From anywhere, deliberately: a node enrols BEFORE it has an address on the private network,
+	// so a rule narrowed to that network would close the door being knocked on.
+	for _, line := range strings.Split(out, "\n") {
+		if strings.Contains(line, "5671") && strings.Contains(line, "saddr") {
+			t.Fatalf("the broker's port is narrowed to the private network, which a machine that "+
+				"has not yet enrolled is not on:\n%s", line)
+		}
+	}
+}
+
+// And a mesh that was never told about a broker still gets a ruleset, rather than an empty one or
+// a panic. A control plane in that state cannot issue tokens either, which is where it surfaces.
+func TestNoBrokerMeansNoSubstrateRuleRatherThanNoRuleset(t *testing.T) {
+	out := AsNftables(nil, []string{"10.42.0.1"}, false, nil)
+	if !strings.Contains(out, "table inet mesh") {
+		t.Fatalf("no ruleset at all:\n%s", out)
+	}
+	if strings.Contains(out, "never derived, never closed\n\t\ttcp dport") {
+		t.Fatalf("a substrate rule was written for a mesh with no broker:\n%s", out)
+	}
+}

internal/catalogue/filtering_test.go

@@ -79,7 +79,7 @@ func TestTwoModulesWantingOnePortAreBothNamed(t *testing.T) {
 		t.Fatalf("a module that wanted this port open is not named: %+v", rules[0])
 	}
 	// The consequence, which is the reason this matters: removing web must not read as closing 443.
-	nft := AsNftables(rules, nil, false)
+	nft := AsNftables(rules, nil, false, nil)
 	if !strings.Contains(nft, "web") || !strings.Contains(nft, "board") {
 		t.Fatalf("the rendered rule set does not name both sources:\n%s", nft)
 	}
@@ -107,7 +107,7 @@ func TestAPortOpenToEveryoneIsNotAlsoRestrictedToTheMesh(t *testing.T) {
 func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "web", Listens: []Listening{{Port: 443, From: FromEverywhere}}},
-	}}, nil), []string{"198.51.100.2"}, false)
+	}}, nil), []string{"198.51.100.2"}, false, nil)
 	// Naming the chain, not just the policy: the forward chain drops too, and an assertion on
 	// "policy drop" alone passes while the input chain accepts everything. It did, once, here.
 	if !strings.Contains(nft, "type filter hook input priority filter; policy drop;") {
@@ -134,7 +134,7 @@ func TestWhatNoModuleDeclaredIsClosed(t *testing.T) {
 // `flush ruleset` would do the first and not the second: it empties every table on the machine,
 // including the ones the container runtime writes for its bridges.
 func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
-	nft := AsNftables(nil, nil, false)
+	nft := AsNftables(nil, nil, false, nil)
 	if strings.Contains(nft, "flush ruleset") {
 		t.Fatalf("loading the rule set empties every table on the machine:\n%s", nft)
 	}
@@ -160,7 +160,7 @@ func TestReloadingReplacesOnlyTheMeshsOwnRules(t *testing.T) {
 // So the chain exists and denies by default, and the runtime's own networks are allowed explicitly
 // — which is how the system being replaced has been doing it on these machines for months.
 func TestWhatIsForwardedIsGovernedToo(t *testing.T) {
-	nft := AsNftables(nil, []string{"198.51.100.2"}, false)
+	nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil)
 	if !strings.Contains(nft, "hook forward priority filter; policy drop") {
 		t.Fatalf("forwarded traffic is not governed, so container ports are open:\n%s", nft)
 	}
@@ -168,7 +168,7 @@ func TestWhatIsForwardedIsGovernedToo(t *testing.T) {
 
 // And containers keep working, which is the whole reason the chain was left out before.
 func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
-	nft := AsNftables(nil, []string{"198.51.100.2"}, false)
+	nft := AsNftables(nil, []string{"198.51.100.2"}, false, nil)
 	for _, network := range []string{"172.16.0.0/12", "192.168.128.0/17"} {
 		if !strings.Contains(nft, "ip saddr "+network+" accept") {
 			t.Fatalf("%s is not allowed, so denying by default stops every container:\n%s", network, nft)
@@ -183,7 +183,7 @@ func TestTheRuntimesOwnNetworksKeepWorking(t *testing.T) {
 func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "web", Listens: []Listening{{Port: 8080, From: FromEverywhere}}},
-	}}, nil), []string{"198.51.100.2"}, false)
+	}}, nil), []string{"198.51.100.2"}, false, nil)
 	if !strings.Contains(nft, "ct original proto-dst 8080 accept") {
 		t.Fatalf("the forwarded rule does not match the port a client asked for:\n%s", nft)
 	}
@@ -193,7 +193,7 @@ func TestAPublishedPortIsMatchedByWhatWasAskedFor(t *testing.T) {
 func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
-	}}, nil), []string{"198.51.100.2"}, false)
+	}}, nil), []string{"198.51.100.2"}, false, nil)
 	if !strings.Contains(nft, "ip saddr { 198.51.100.2 } ct original proto-dst 5432 accept") {
 		t.Fatalf("a mesh-only port is reachable from anywhere once forwarded:\n%s", nft)
 	}
@@ -203,7 +203,7 @@ func TestAMeshScopedPortIsMeshScopedWhenForwarded(t *testing.T) {
 func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
-	}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false)
+	}}, nil), []string{"198.51.100.2", "198.51.100.3"}, false, nil)
 	if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 5432 accept") {
 		t.Fatalf("a mesh-scoped port was not restricted to the mesh's addresses:\n%s", nft)
 	}
@@ -213,7 +213,7 @@ func TestFromTheMeshIsTheNodesTheMeshKnows(t *testing.T) {
 func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
-	}}, nil), nil, false)
+	}}, nil), nil, false, nil)
 	if strings.Contains(nft, "dport 5432 accept") {
 		t.Fatalf("a port meant for the mesh was opened to everything:\n%s", nft)
 	}
@@ -226,7 +226,7 @@ func TestAMeshPortOnANodeWithNoMeshIsClosedAndSaysSo(t *testing.T) {
 func TestAMachineScopedPortIsNotOpened(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "cache", Listens: []Listening{{Port: 6379, From: FromMachine}}},
-	}}, nil), []string{"198.51.100.2"}, false)
+	}}, nil), []string{"198.51.100.2"}, false, nil)
 	if strings.Contains(nft, "dport 6379 accept") {
 		t.Fatalf("a port for this machine only was opened to the network:\n%s", nft)
 	}
@@ -268,7 +268,7 @@ func TestAskingForTheRuleSetWithNowhereToPutItIsRefused(t *testing.T) {
 func TestAMeshOnBothAddressFamiliesRendersBoth(t *testing.T) {
 	nft := AsNftables(mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "store", Listens: []Listening{{Port: 5432, From: FromMesh}}},
-	}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false)
+	}}, nil), []string{"198.51.100.2", "2001:db8::2"}, false, nil)
 	if !strings.Contains(nft, "ip saddr { 198.51.100.2 } tcp dport 5432 accept") {
 		t.Fatalf("the machines with v4 addresses were dropped:\n%s", nft)
 	}
@@ -672,7 +672,7 @@ func TestExposureRefusesAPortNotListenedOnAndABadSource(t *testing.T) {
 // loading the rules lives on conntrack until it drops, and then the machine is reached from a
 // rescue console (novox/hq issue 047).
 func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
-	nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false)
+	nft := AsNftables(nil, []string{"198.51.100.2", "198.51.100.3"}, false, nil)
 	if !strings.Contains(nft, "ip saddr { 198.51.100.2, 198.51.100.3 } tcp dport 22 accept") {
 		t.Fatalf("ssh is not open to the mesh, so a machine can lock everyone out:\n%s", nft)
 	}
@@ -685,7 +685,7 @@ func TestSSHIsOpenFromTheMeshEvenWhenNothingIsAssigned(t *testing.T) {
 // And from outside as well, on a machine that faces outward — because that is the way in when the
 // private network is the thing that broke.
 func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
-	nft := AsNftables(nil, []string{"198.51.100.2"}, true)
+	nft := AsNftables(nil, []string{"198.51.100.2"}, true, nil)
 	if !strings.Contains(nft, "\t\ttcp dport 22 accept") {
 		t.Fatalf("a machine reachable from outside does not answer ssh there:\n%s", nft)
 	}
@@ -697,7 +697,7 @@ func TestSSHIsOpenFromOutsideOnAMachineThatFacesIt(t *testing.T) {
 // to narrow the rule to, so narrowing it shuts the port entirely — on the first machine anybody
 // adopts, reached over the network, closed by the act of adopting it.
 func TestSSHIsNeverLeftWithoutARule(t *testing.T) {
-	nft := AsNftables(nil, nil, false)
+	nft := AsNftables(nil, nil, false, nil)
 	if !strings.Contains(nft, "tcp dport 22 accept") {
 		t.Fatalf("a machine with no mesh addresses has no ssh rule, so adopting it locks it:\n%s", nft)
 	}

internal/catalogue/print_rehearsal_test.go

@@ -12,5 +12,5 @@ func TestPrintRehearsalRuleset(t *testing.T) {
 	rules := mustFilter(t, Resolution{Modules: []Manifest{
 		{Module: "pub", Listens: []Listening{{Port: 8099, From: FromMesh, Why: "the thing it serves"}}},
 	}}, nil)
-	t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true))
+	t.Log("\n" + AsNftables(rules, []string{"192.0.2.20"}, true, nil))
 }