mesh-controller/internal/overlay/graph.go

// Package overlay computes the private network every node runs on.
//
// novox/hq 08-connectivity. This is control-plane work by definition: a peer list is derived from
// every node at once, and no node has that. A node computes nothing about the mesh — it generates
// a keypair, publishes the public half, and receives the rest.
//
// The shape is a hub, with direct peering between nodes at the same site. Not a full mesh, and
// the reason is a property of WireGuard rather than a preference: there is no failover. A more
// specific route to a dead endpoint blackholes; it does not fall back to the general one. So a
// node gets exactly one path to any peer, because two would mean one of them silently swallowing
// traffic.
package overlay

import (
	"errors"
	"fmt"
	"sort"
	"strings"
)

// Node is one machine's place on the network, as the mesh holds it.
type Node struct {
	Name     string
	Key      string
	Endpoint string
	Site     string
	Hub      bool
	Address  string

	// Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the
	// mesh has no record of, each known by the public key and the address the found tunnel routed
	// to it. Only a hub has any. They stay in its peer list until a node enrols with that key —
	// from then on the node is the peer.
	Carried []Carried
	// TakesOver names the found tunnel this node's private network replaces: its unit is stopped
	// and disabled, never flushed, and its configuration kept, before the mesh's interface comes
	// up with the found key. Nil on a node that raises the mesh's interface beside whatever it has.
	TakesOver *TakeOver
}

// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a
// node of the mesh.
type Carried struct {
	Key     string
	Address string
}

// TakeOver is the found tunnel a node's private network takes over, as the host is told it.
type TakeOver struct {
	Interface string
	Unit      string
	Config    string
	// MTU is the found tunnel's, when it set one — emitted so a tuned path keeps its MTU.
	MTU int
	// Port is the port the found tunnel listened on. The mesh's interface must listen on it too,
	// even on a node that is not dialable from the hub: a home node's LAN peers dial it there
	// (novox/hq: a taken tunnel brings its port). Listening is "a peer dials me here"; it is not
	// "the hub can dial me", which is Reachable — the two were conflated.
	Port int
}

// HostPrefix is one address as a route: /32 for IPv4, /128 for IPv6.
func HostPrefix(address string) string {
	if strings.Contains(address, ":") {
		return address + "/128"
	}
	return address + "/32"
}

// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named
// by the key its packets arrive under.
func CarriedName(key string) string {
	short := key
	if len(short) > 8 {
		short = short[:8] + "…"
	}
	return "a peer of the tunnel (" + short + ")"
}

// Reachable reports whether other nodes can dial this one. Declared, never inferred.
func (n Node) Reachable() bool { return strings.TrimSpace(n.Endpoint) != "" }

// Peer is one entry in a node's peer list.
type Peer struct {
	Name string
	Key  string
	// Endpoint is empty when this peer cannot be dialled — it must dial us instead.
	Endpoint string
	// Allowed is what traffic goes down this tunnel. A single address for a direct peer; the
	// whole overlay for the hub, which is what makes it the route of last resort.
	Allowed string
	// Keepalive matters only on the side behind NAT: a node that cannot be dialled has to keep
	// the path open from its end, or the peer's first packet arrives at a mapping that has
	// already expired.
	Keepalive bool
	// Why this peer is in the list, for a person reading a generated file and wondering.
	Why string
}

// Graph is every node's peer list.
type Graph map[string][]Peer

// ErrNoHub means nobody has said which node is the hub.
//
// Its own error rather than an empty graph: a mesh with no hub has no path between sites, and
// answering with "no peers" would look like a working mesh where nothing can reach anything.
var ErrNoHub = errors.New("this mesh has no hub, so there is no path between sites")

// Compute derives every node's peer list.
//
// Nodes without a key or an address are skipped rather than refused: a node that has enrolled and
// not yet been given a place on the network is an ordinary in-between state, and failing the whole
// graph because one node is half-configured would mean no node gets a network.
func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
	var hub *Node
	usable := make([]Node, 0, len(nodes))
	for i := range nodes {
		n := nodes[i]
		if n.Key == "" || n.Address == "" {
			continue
		}
		usable = append(usable, n)
		if n.Hub {
			hub = &usable[len(usable)-1]
		}
	}
	if len(usable) == 0 {
		return Graph{}, nil
	}
	if hub == nil {
		return nil, ErrNoHub
	}
	if !hub.Reachable() {
		return nil, fmt.Errorf(
			"%s is the hub and has no endpoint, so nothing can dial it. The hub is the one node "+
				"that must be reachable from wherever the others are", hub.Name)
	}

	graph := Graph{}
	for _, self := range usable {
		var peers []Peer

		// A node may share a site with the hub, and then the hub is one peer rather than two.
		// Written before the loop because it changes what that loop may emit: WireGuard takes one
		// entry per public key, so a hub appearing twice is a configuration it refuses — and the
		// mesh would have produced it silently. The lab found this on the first two machines that
		// shared a site with their hub.
		hubIsHere := !self.Hub && self.Site != "" && self.Site == hub.Site

		for _, other := range usable {
			if other.Name == self.Name || (hubIsHere && other.Name == hub.Name) {
				continue
			}
			// Two nodes at the same site peer directly — but only if one of them can be
			// dialled. If neither can, nobody opens the path, and the direct route is more
			// specific than the hub's, so it wins and blackholes. That is this design's own
			// stated hazard arriving in it: *a more specific route to a dead endpoint
			// blackholes; it does not fall back to the general one.*
			//
			// Found in the lab with two machines at one site behind no reachable address, which
			// is the ordinary shape of a home: they were given each other as peers, neither
			// could start, and they could not reach each other at all while both reached the hub
			// perfectly.
			if self.Site != "" && self.Site == other.Site && (self.Reachable() || other.Reachable()) {
				peers = append(peers, Peer{
					Name: other.Name, Key: other.Key,
					Endpoint:  other.Endpoint,
					Allowed:   other.Address + "/32",
					Keepalive: !self.Reachable(),
					Why:       "at the same site",
				})
			}
		}

		if !self.Hub {
			// Everything else goes through the hub, including a node that roams. AllowedIPs is
			// the whole overlay, so this is the route of last resort — and because direct peers
			// above are single addresses, they win on specificity without either being ambiguous.
			why := "the hub — everything not at this site"
			if hubIsHere {
				// One entry doing both jobs: the direct path to a machine that happens to be
				// here, and the route to everywhere else. Splitting them would need two entries
				// for one key, which is the thing being avoided.
				why = "the hub, which is also at this site — everything goes here"
			}
			peers = append(peers, Peer{
				Name: hub.Name, Key: hub.Key,
				Endpoint:  hub.Endpoint,
				Allowed:   overlayCIDR,
				Keepalive: !self.Reachable(),
				Why:       why,
			})
		} else {
			// The hub holds every node that does not share a site with it, because those nodes
			// route through it and it must know where to send the replies. Ones it cannot dial
			// will dial it.
			enrolled := map[string]bool{}
			for _, other := range usable {
				enrolled[other.Key] = true
				if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
					continue
				}
				peers = append(peers, Peer{
					Name: other.Name, Key: other.Key,
					Endpoint: other.Endpoint,
					Allowed:  other.Address + "/32",
					Why:      "routes through this hub",
				})
			}
			// And every peer of the tunnel it took over that has not enrolled (novox/hq ADR
			// 0105): the same key and the same address the found tunnel had for it, so the
			// machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in,
			// as it always did. Once a node enrols with that key, the node's entry above is the
			// peer, and WireGuard takes one entry per key.
			for _, c := range self.Carried {
				if enrolled[c.Key] {
					continue
				}
				peers = append(peers, Peer{
					Name: CarriedName(c.Key), Key: c.Key,
					Allowed: HostPrefix(c.Address),
					Why:     "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh",
				})
			}
		}

		sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
		graph[self.Name] = peers
	}
	return graph, nil
}