A working private network, and four reasons it did not work

Three machines across two sites, two of them behind no reachable address, all
nine paths open. The mesh computes the graph, delivers it as a declaration, and
the nodes bring it up.

Every fault below looked like success from inside the mesh: the graph was
right, the files were right, the services were up, every node reported it had
applied. None was reachable by reasoning.

A running interface does not re-read its configuration. A node joins, every
existing node's peer list changes, the file is replaced -- and the service is
already running, so nothing reloads it. Fixed as declared state rather than a
command: the service must reflect the file. A command to restart would be an
action, and the link may not carry one. The host refused exactly that, which is
how this shape was arrived at.

A hub sharing a site with a spoke appeared twice in that spoke's peer list --
once as a direct peer, once as the route of last resort. WireGuard takes one
entry per key and refuses the file. The ordinary shape of a small mesh, and in
none of the tests written before it ran.

Two nodes at one site that neither can be dialled were peered directly. Nobody
opens the path, and the direct route is more specific than the hub's, so it
wins and blackholes -- this design's own warning arriving in its
implementation. They now route through the hub unless one end can be dialled.

And Docker sets the FORWARD policy to DROP, so a hub with ip_forward enabled
carried nothing between its spokes. The substrate at tier 1 silently breaks the
network at tier 2, and nothing in either tier's state says so. The hub inserts
its own rule above those chains and removes it on the way down.

Two weak tests found by injection along the way: one asserted the keepalive
rule only against the hub, whose peer entries happen not to set that field at
all, so it tested an absence; the other checked the firewall rules by looking
for FORWARD anywhere, which the PostDown line satisfies on its own.

internal/inventory/addresses.go

@@ -0,0 +1,76 @@
+package inventory
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+)
+
+// The mesh assigns overlay addresses. A node computes nothing about the network it is joining —
+// it generates a keypair, publishes the public half, and receives the rest
+// (novox/hq 08-connectivity).
+
+// AssignAddress gives a node its place on the private network, if it has none.
+//
+// Idempotent: a node that already has an address keeps it. Reassigning would change every other
+// node's peer list to chase it, and an address that moves is the thing declaring the hub was
+// meant to stop.
+//
+// Allocated in order from the range, taking the lowest free one. Not random: a person reading a
+// peer list should be able to guess which node an address belongs to, and reuse of a released
+// address is a smaller problem than a list nobody can hold in their head.
+func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) {
+	prefix, err := netip.ParsePrefix(cidr)
+	if err != nil {
+		return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err)
+	}
+
+	var existing *string
+	if err := i.store.Pool().QueryRow(ctx,
+		`select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil {
+		return "", err
+	}
+	if existing != nil && *existing != "" {
+		return *existing, nil
+	}
+
+	taken := map[string]bool{}
+	rows, err := i.store.Pool().Query(ctx,
+		`select host(overlay_address) from node where overlay_address is not null`)
+	if err != nil {
+		return "", err
+	}
+	for rows.Next() {
+		var a string
+		if err := rows.Scan(&a); err != nil {
+			rows.Close()
+			return "", err
+		}
+		taken[a] = true
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		return "", err
+	}
+
+	// The first address in a range is conventionally the network itself and is skipped, so
+	// allocation starts one past it.
+	candidate := prefix.Masked().Addr().Next()
+	for prefix.Contains(candidate) {
+		if !taken[candidate.String()] {
+			if _, err := i.store.Pool().Exec(ctx,
+				`update node set overlay_address = $2::inet where id = $1`,
+				node, candidate.String()); err != nil {
+				return "", err
+			}
+			return candidate.String(), nil
+		}
+		candidate = candidate.Next()
+	}
+	// Said plainly rather than returning an empty address that fails later on a machine. A mesh
+	// that has outgrown its range needs a person, and renumbering is not something to attempt
+	// halfway through assigning one node.
+	return "", fmt.Errorf(
+		"every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+
+			"range and renumbering it is a deliberate act", cidr, node)
+}

internal/link/enrolment.go

@@ -32,8 +32,8 @@ type Enrolment struct {
 // single statement that both finds and marks it, so two machines racing on one secret produce one
 // winner. Only then is a key recorded — because recording a key for a node whose token turned out
 // to be spent would leave the mesh believing a machine that never had the right to join.
-func (e Enrolment) Enrol(ctx context.Context, secret string, public ed25519.PublicKey,
-	profile map[string]any) (EnrolReply, error) {
+func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) {
+	secret, public, profile := request.Secret, ed25519.PublicKey(request.PublicKey), request.Profile
 
 	if len(public) != ed25519.PublicKeySize {
 		return EnrolReply{}, fmt.Errorf("a node presented a %d-byte key, and an identity is %d",
@@ -84,6 +84,17 @@ func (e Enrolment) Enrol(ctx context.Context, secret string, public ed25519.Publ
 		reply.Password = password
 	}
 
+	// Recorded before the profile because the overlay is the first declaration this node will
+	// receive, and without this key the mesh cannot compose one. A node enrolled with no overlay
+	// key is a node the graph skips — an ordinary in-between state, and one worth leaving as
+	// briefly as possible.
+	if request.OverlayKey != "" {
+		if err := e.Inventory.RecordOverlayKey(ctx, node.ID, request.OverlayKey); err != nil {
+			return EnrolReply{}, fmt.Errorf(
+				"the token was spent and %s's overlay key could not be recorded: %w", node.Name, err)
+		}
+	}
+
 	if profile != nil {
 		// Not fatal if it fails. The profile is what the control plane needs in order to decide
 		// what this machine should run, and it is reported again on every connection — so losing

internal/link/protocol.go

@@ -40,6 +40,10 @@ type EnrolRequest struct {
 	// half has never left that machine (novox/hq ADR 0004).
 	PublicKey []byte `json:"public_key"`
 
+	// OverlayKey is the public half of this node's key on the private network — a different key
+	// from PublicKey, and the mesh only ever sees this half.
+	OverlayKey string `json:"overlay_key,omitempty"`
+
 	// Profile is what this machine can be asked to do. The control plane cannot decide what a
 	// node should run without it, so it arrives with enrolment rather than being asked for after.
 	Profile map[string]any `json:"profile,omitempty"`

internal/link/serve.go

@@ -2,7 +2,6 @@ package link
 
 import (
 	"context"
-	"crypto/ed25519"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -24,7 +23,7 @@ const AMQPVar = "MESH_BROKER_AMQP"
 type Enroller interface {
 	// Enrol spends the token, records the key, and reports the node's name. The error is
 	// returned to the node as a refusal; it must be the same for every reason a token can fail.
-	Enrol(ctx context.Context, secret string, public ed25519.PublicKey, profile map[string]any) (EnrolReply, error)
+	Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error)
 }
 
 // Server consumes what nodes say.
@@ -194,7 +193,7 @@ func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) {
 	if err := json.Unmarshal(delivery.Body, &request); err != nil {
 		s.log.Printf("an enrolment request could not be read: %v", err)
 	} else {
-		accepted, err := s.enroller.Enrol(ctx, request.Secret, request.PublicKey, request.Profile)
+		accepted, err := s.enroller.Enrol(ctx, request)
 		if err != nil {
 			// Logged in full here, where an operator can see it; sent back as one refusal, so
 			// that somebody guessing learns nothing from which reason came back.

internal/overlay/declaration.go

@@ -62,6 +62,19 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
 			// be told again. A node whose overlay only exists while something is watching is not
 			// a node that survives being switched off and on.
 			"boot": "enabled",
+			// And restarted when the peer list changes, because a running interface does not
+			// re-read its configuration.
+			//
+			// This is the whole of it: a node joins, every existing node's peer list changes,
+			// each file is replaced — and without this the service is already running, nothing
+			// reloads it, and every node keeps a network that no longer matches the mesh. It
+			// reports complete success. The lab found it the moment a third node arrived.
+			//
+			// Declared state rather than a command. The service must reflect the file; the host
+			// works out that it does not. A command to restart would be an action, and the link
+			// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
+			// which is how this shape was arrived at.
+			"restart-on": []string{"overlay-config"},
 		},
 	}
 
@@ -96,6 +109,39 @@ func config(node Node, peers []Peer, keyPath string) string {
 	// travelled. Everything else in this file came from the mesh; this one line is the node's.
 	fmt.Fprintf(&b, "PostUp = wg set %%i private-key %s\n", keyPath)
 
+	if node.Hub {
+		// A hub carries traffic *between* its spokes, and a Linux machine does not forward
+		// packets unless it is told to. Without this every spoke reaches the hub perfectly and
+		// no spoke reaches any other — which is exactly how it failed in the lab, and it looks
+		// like a peering problem rather than a kernel setting.
+		//
+		// Here rather than in a separate resource because it is part of what being a hub means,
+		// and because it should last exactly as long as the interface does: a machine that stops
+		// being the hub should stop forwarding, and PostDown below is how that happens.
+		b.WriteString("PostUp = sysctl -q -w net.ipv4.ip_forward=1\n")
+		b.WriteString("PostDown = sysctl -q -w net.ipv4.ip_forward=0\n")
+
+		// And past the machine's own firewall, which on any node with a container runtime is
+		// closed. Docker sets the FORWARD policy to DROP and inserts its chains, so the substrate
+		// this mesh installs at tier 1 silently breaks the network it builds at tier 2: every
+		// spoke reaches the hub, no spoke reaches any other, and every part of it reports
+		// success. Found in the lab; nothing about it is visible from the mesh's own state.
+		//
+		// Inserted at the top so it precedes those chains, and removed on the way down so a
+		// machine that stops being the hub stops carrying other people's traffic. Guarded on
+		// iptables existing: a machine without it has no policy to get past.
+		for _, direction := range []string{"-i", "-o"} {
+			fmt.Fprintf(&b,
+				"PostUp = command -v iptables >/dev/null && iptables -I FORWARD 1 %s %%i -j ACCEPT || true\n",
+				direction)
+		}
+		for _, direction := range []string{"-i", "-o"} {
+			fmt.Fprintf(&b,
+				"PostDown = command -v iptables >/dev/null && iptables -D FORWARD %s %%i -j ACCEPT || true\n",
+				direction)
+		}
+	}
+
 	for _, p := range peers {
 		fmt.Fprintf(&b, "\n# %s — %s\n[Peer]\n", p.Name, p.Why)
 		fmt.Fprintf(&b, "PublicKey = %s\n", p.Key)

internal/overlay/declaration_test.go

@@ -2,6 +2,7 @@ package overlay
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 )
@@ -137,3 +138,82 @@ func TestOnlyAReachableNodeListens(t *testing.T) {
 		t.Error("a reachable node does not listen on the port its endpoint names")
 	}
 }
+
+func TestTheServiceIsRestartedWhenThePeerListChanges(t *testing.T) {
+	// The fault this exists to catch, found in the lab the moment a third node arrived: a running
+	// WireGuard interface does not re-read its configuration. The file was replaced, the service
+	// was already running so nothing reloaded it, and every existing node kept a network that no
+	// longer matched the mesh — while looking entirely successful.
+	//
+	// So the declaration has to verify what the interface is *carrying*, not what the file says.
+	//
+	// And it must be declared state rather than a command: the host refuses an action arriving
+	// over the link (novox/hq ADR 0005), correctly, which is how this shape was arrived at. There
+	// is a test below that no action is ever in here.
+	_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"},
+		[]Peer{{Name: "anchor", Key: "HUBKEY", Allowed: "10.42.0.0/16"}})
+
+	for _, r := range resources {
+		if r["type"] != "service" {
+			continue
+		}
+		reflects := fmt.Sprint(r["restart-on"])
+		if !strings.Contains(reflects, "overlay-config") {
+			t.Errorf("the interface does not restart when its configuration changes: %v", reflects)
+		}
+		return
+	}
+	t.Error("nothing in the declaration brings the interface up")
+}
+
+func TestTheOverlayDeclarationCarriesNoAction(t *testing.T) {
+	// The link may not carry an action, and the host refuses a declaration containing one — whole,
+	// not in part. An overlay declaration with an action in it does not half-apply: it leaves the
+	// node with no network at all.
+	_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
+	for _, r := range resources {
+		if r["type"] == "action" {
+			t.Errorf("the declaration contains an action (%v); the host will refuse the whole "+
+				"thing and the node will have no network", r["id"])
+		}
+	}
+}
+
+func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) {
+	// A hub carries traffic between its spokes, and Linux does not forward packets unless told
+	// to. Without it every spoke reaches the hub perfectly and no spoke reaches any other — which
+	// is how it failed in the lab, and it presents as a peering problem rather than a kernel
+	// setting, so it is worth being sure of.
+	hub, _ := declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
+		Endpoint: "198.51.100.1:51820", Hub: true}, nil)
+	if !strings.Contains(hub, "ip_forward=1") {
+		t.Error("the hub does not enable forwarding, so its spokes cannot reach each other")
+	}
+	if !strings.Contains(hub, "ip_forward=0") {
+		t.Error("the hub never stops forwarding; a machine that stops being the hub would keep " +
+			"passing traffic it is no longer part of")
+	}
+
+	// And past the machine's own firewall. Any node with a container runtime has FORWARD set to
+	// DROP by Docker, so enabling ip_forward alone changes nothing — which is exactly how it
+	// failed, with every spoke reaching the hub and no spoke reaching any other.
+	// Checked as PostUp specifically. "contains FORWARD" passes on the PostDown line alone,
+	// which would leave a hub that tears down rules it never put up.
+	if !strings.Contains(hub, "PostUp = command -v iptables") {
+		t.Error("the hub does not open its own firewall, so a container runtime's DROP policy " +
+			"silently eats everything it was supposed to carry")
+	}
+	if !strings.Contains(hub, "PostDown = command -v iptables") {
+		t.Error("the hub never removes those rules, so a machine that stops being the hub keeps " +
+			"passing traffic it is no longer part of")
+	}
+
+	spoke, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
+	if strings.Contains(spoke, "FORWARD") {
+		t.Error("a spoke was given forwarding rules it has no use for")
+	}
+	if strings.Contains(spoke, "ip_forward") {
+		t.Error("a spoke was told to forward packets, which is not its job and widens what a " +
+			"compromised one could do")
+	}
+}

internal/overlay/graph.go

@@ -91,14 +91,28 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
 	for _, self := range usable {
 		var peers []Peer
 
+		// A node may share a site with the hub, and then the hub is one peer rather than two.
+		// Written before the loop because it changes what that loop may emit: WireGuard takes one
+		// entry per public key, so a hub appearing twice is a configuration it refuses — and the
+		// mesh would have produced it silently. The lab found this on the first two machines that
+		// shared a site with their hub.
+		hubIsHere := !self.Hub && self.Site != "" && self.Site == hub.Site
+
 		for _, other := range usable {
-			if other.Name == self.Name {
+			if other.Name == self.Name || (hubIsHere && other.Name == hub.Name) {
 				continue
 			}
-			// Two nodes at the same site peer directly. A site is where a machine physically is,
-			// and machines that share one have a path that does not need the hub — so using it
-			// keeps their traffic off a link that may be somewhere else entirely.
-			if self.Site != "" && self.Site == other.Site {
+			// Two nodes at the same site peer directly — but only if one of them can be
+			// dialled. If neither can, nobody opens the path, and the direct route is more
+			// specific than the hub's, so it wins and blackholes. That is this design's own
+			// stated hazard arriving in it: *a more specific route to a dead endpoint
+			// blackholes; it does not fall back to the general one.*
+			//
+			// Found in the lab with two machines at one site behind no reachable address, which
+			// is the ordinary shape of a home: they were given each other as peers, neither
+			// could start, and they could not reach each other at all while both reached the hub
+			// perfectly.
+			if self.Site != "" && self.Site == other.Site && (self.Reachable() || other.Reachable()) {
 				peers = append(peers, Peer{
 					Name: other.Name, Key: other.Key,
 					Endpoint:  other.Endpoint,
@@ -113,12 +127,19 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
 			// Everything else goes through the hub, including a node that roams. AllowedIPs is
 			// the whole overlay, so this is the route of last resort — and because direct peers
 			// above are single addresses, they win on specificity without either being ambiguous.
+			why := "the hub — everything not at this site"
+			if hubIsHere {
+				// One entry doing both jobs: the direct path to a machine that happens to be
+				// here, and the route to everywhere else. Splitting them would need two entries
+				// for one key, which is the thing being avoided.
+				why = "the hub, which is also at this site — everything goes here"
+			}
 			peers = append(peers, Peer{
 				Name: hub.Name, Key: hub.Key,
 				Endpoint:  hub.Endpoint,
 				Allowed:   overlayCIDR,
 				Keepalive: !self.Reachable(),
-				Why:       "the hub — everything not at this site",
+				Why:       why,
 			})
 		} else {
 			// The hub holds every node that does not share a site with it, because those nodes

internal/overlay/graph_test.go

@@ -222,3 +222,85 @@ func TestNobodyPeersWithThemselves(t *testing.T) {
 		}
 	}
 }
+
+func TestAHubAtYourOwnSiteAppearsOnceNotTwice(t *testing.T) {
+	// WireGuard takes one entry per public key. A hub that shares a site with a spoke was being
+	// emitted twice — once as a direct peer and once as the route of last resort — producing a
+	// configuration the interface refuses, from a mesh that thought it had succeeded.
+	//
+	// Found in the lab on the first two machines that shared a site with their hub, which is the
+	// ordinary case for a small mesh and was in none of the tests above.
+	g, err := Compute([]Node{
+		at("anchor", "lab", "10.42.0.1", "192.0.2.10:51820", true),
+		at("laptop", "lab", "10.42.0.2", "", false),
+	}, cidr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	seen := map[string]int{}
+	for _, p := range g["laptop"] {
+		seen[p.Key]++
+	}
+	for key, count := range seen {
+		if count > 1 {
+			t.Errorf("the key %s appears %d times; WireGuard takes one entry per key", key, count)
+		}
+	}
+
+	if len(g["laptop"]) != 1 {
+		t.Fatalf("laptop has %d peers; the hub at its own site is one peer, not two", len(g["laptop"]))
+	}
+	// And that single entry has to carry everything, or the node keeps a direct path to the hub
+	// and loses its route to everywhere else.
+	if got := g["laptop"][0].Allowed; got != cidr {
+		t.Errorf("the single entry allows %s; it is both the direct path and the route of last "+
+			"resort, so it carries the whole overlay", got)
+	}
+}
+
+func TestTwoUnreachableNodesAtOneSiteDoNotPeerDirectly(t *testing.T) {
+	// Nobody would open the path, and the direct route is more specific than the hub's — so it
+	// wins and blackholes. Both nodes would reach the hub perfectly and be unable to reach each
+	// other, which is the worst arrangement available: it looks configured and is not.
+	//
+	// This is the design's own warning arriving in its implementation, and the lab found it with
+	// two machines at one site behind no reachable address — the ordinary shape of a house.
+	g, err := Compute([]Node{
+		at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
+		at("desk", "house", "10.42.0.2", "", false),
+		at("server", "house", "10.42.0.3", "", false),
+	}, cidr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := peersOf(t, g, "desk")["server"]; ok {
+		t.Error("two nodes that neither can be dialled were peered directly; neither could open " +
+			"the path and the route is more specific than the hub's, so it blackholes")
+	}
+	// And they must still be able to reach each other — through the hub.
+	if hub := peersOf(t, g, "desk")["anchor"]; hub.Allowed != cidr {
+		t.Errorf("desk's route to everything else allows %s", hub.Allowed)
+	}
+}
+
+func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) {
+	// The other side of it: if either end can be dialled, the path can be opened, and using it
+	// keeps their traffic off a hub that may be on another continent.
+	g, err := Compute([]Node{
+		at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
+		at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
+		at("server", "house", "10.42.0.3", "", false),
+	}, cidr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := peersOf(t, g, "server")["desk"]; !ok {
+		t.Error("a node did not peer with a reachable neighbour at its own site")
+	}
+	if _, ok := peersOf(t, g, "desk")["server"]; !ok {
+		t.Error("the reachable node did not hold its unreachable neighbour, so replies have " +
+			"nowhere to go")
+	}
+}