// Package overlay computes the private network every node runs on. // // novox/hq 08-connectivity. This is control-plane work by definition: a peer list is derived from // every node at once, and no node has that. A node computes nothing about the mesh — it generates // a keypair, publishes the public half, and receives the rest. // // The shape is a hub, with direct peering between nodes at the same site. Not a full mesh, and // the reason is a property of WireGuard rather than a preference: there is no failover. A more // specific route to a dead endpoint blackholes; it does not fall back to the general one. So a // node gets exactly one path to any peer, because two would mean one of them silently swallowing // traffic. package overlay import ( "errors" "fmt" "sort" "strings" ) // Node is one machine's place on the network, as the mesh holds it. type Node struct { Name string Key string Endpoint string Site string Hub bool Address string // Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the // mesh has no record of, each known by the public key and the address the found tunnel routed // to it. Only a hub has any. They stay in its peer list until a node enrols with that key — // from then on the node is the peer. Carried []Carried // TakesOver names the found tunnel this node's private network replaces: its unit is stopped // and disabled, never flushed, and its configuration kept, before the mesh's interface comes // up with the found key. Nil on a node that raises the mesh's interface beside whatever it has. TakesOver *TakeOver } // Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a // node of the mesh. type Carried struct { Key string Address string } // TakeOver is the found tunnel a node's private network takes over, as the host is told it. type TakeOver struct { Interface string Unit string Config string } // CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named // by the key its packets arrive under. func CarriedName(key string) string { short := key if len(short) > 8 { short = short[:8] + "…" } return "a peer of the tunnel (" + short + ")" } // Reachable reports whether other nodes can dial this one. Declared, never inferred. func (n Node) Reachable() bool { return strings.TrimSpace(n.Endpoint) != "" } // Peer is one entry in a node's peer list. type Peer struct { Name string Key string // Endpoint is empty when this peer cannot be dialled — it must dial us instead. Endpoint string // Allowed is what traffic goes down this tunnel. A single address for a direct peer; the // whole overlay for the hub, which is what makes it the route of last resort. Allowed string // Keepalive matters only on the side behind NAT: a node that cannot be dialled has to keep // the path open from its end, or the peer's first packet arrives at a mapping that has // already expired. Keepalive bool // Why this peer is in the list, for a person reading a generated file and wondering. Why string } // Graph is every node's peer list. type Graph map[string][]Peer // ErrNoHub means nobody has said which node is the hub. // // Its own error rather than an empty graph: a mesh with no hub has no path between sites, and // answering with "no peers" would look like a working mesh where nothing can reach anything. var ErrNoHub = errors.New("this mesh has no hub, so there is no path between sites") // Compute derives every node's peer list. // // Nodes without a key or an address are skipped rather than refused: a node that has enrolled and // not yet been given a place on the network is an ordinary in-between state, and failing the whole // graph because one node is half-configured would mean no node gets a network. func Compute(nodes []Node, overlayCIDR string) (Graph, error) { var hub *Node usable := make([]Node, 0, len(nodes)) for i := range nodes { n := nodes[i] if n.Key == "" || n.Address == "" { continue } usable = append(usable, n) if n.Hub { hub = &usable[len(usable)-1] } } if len(usable) == 0 { return Graph{}, nil } if hub == nil { return nil, ErrNoHub } if !hub.Reachable() { return nil, fmt.Errorf( "%s is the hub and has no endpoint, so nothing can dial it. The hub is the one node "+ "that must be reachable from wherever the others are", hub.Name) } graph := Graph{} for _, self := range usable { var peers []Peer // A node may share a site with the hub, and then the hub is one peer rather than two. // Written before the loop because it changes what that loop may emit: WireGuard takes one // entry per public key, so a hub appearing twice is a configuration it refuses — and the // mesh would have produced it silently. The lab found this on the first two machines that // shared a site with their hub. hubIsHere := !self.Hub && self.Site != "" && self.Site == hub.Site for _, other := range usable { if other.Name == self.Name || (hubIsHere && other.Name == hub.Name) { continue } // Two nodes at the same site peer directly — but only if one of them can be // dialled. If neither can, nobody opens the path, and the direct route is more // specific than the hub's, so it wins and blackholes. That is this design's own // stated hazard arriving in it: *a more specific route to a dead endpoint // blackholes; it does not fall back to the general one.* // // Found in the lab with two machines at one site behind no reachable address, which // is the ordinary shape of a home: they were given each other as peers, neither // could start, and they could not reach each other at all while both reached the hub // perfectly. if self.Site != "" && self.Site == other.Site && (self.Reachable() || other.Reachable()) { peers = append(peers, Peer{ Name: other.Name, Key: other.Key, Endpoint: other.Endpoint, Allowed: other.Address + "/32", Keepalive: !self.Reachable(), Why: "at the same site", }) } } if !self.Hub { // Everything else goes through the hub, including a node that roams. AllowedIPs is // the whole overlay, so this is the route of last resort — and because direct peers // above are single addresses, they win on specificity without either being ambiguous. why := "the hub — everything not at this site" if hubIsHere { // One entry doing both jobs: the direct path to a machine that happens to be // here, and the route to everywhere else. Splitting them would need two entries // for one key, which is the thing being avoided. why = "the hub, which is also at this site — everything goes here" } peers = append(peers, Peer{ Name: hub.Name, Key: hub.Key, Endpoint: hub.Endpoint, Allowed: overlayCIDR, Keepalive: !self.Reachable(), Why: why, }) } else { // The hub holds every node that does not share a site with it, because those nodes // route through it and it must know where to send the replies. Ones it cannot dial // will dial it. enrolled := map[string]bool{} for _, other := range usable { enrolled[other.Key] = true if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) { continue } peers = append(peers, Peer{ Name: other.Name, Key: other.Key, Endpoint: other.Endpoint, Allowed: other.Address + "/32", Why: "routes through this hub", }) } // And every peer of the tunnel it took over that has not enrolled (novox/hq ADR // 0105): the same key and the same address the found tunnel had for it, so the // machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in, // as it always did. Once a node enrols with that key, the node's entry above is the // peer, and WireGuard takes one entry per key. for _, c := range self.Carried { if enrolled[c.Key] { continue } peers = append(peers, Peer{ Name: CarriedName(c.Key), Key: c.Key, Allowed: c.Address + "/32", Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh", }) } } sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name }) graph[self.Name] = peers } return graph, nil }