The mesh computes a private network it cannot impersonate
The first thing the control plane decides rather than relays. Every node's peer list is derived from every node at once, which is what makes this control-plane work by definition: no node has that view. A hub, with direct peering between nodes at the same site. Not a full mesh, and the reason is a property of WireGuard rather than a preference -- there is no failover, so a more specific route to a dead endpoint blackholes instead of falling back. A node gets exactly one path to any peer, because two would mean one of them silently swallowing traffic. A roaming node is hub-only for the same reason. Reachability and the hub are declared, never inferred from an address. The address is evidence and is not the fact: carrier-grade NAT looks public and is not, a routable address behind a closed firewall looks public and is not, and the regular expression that used to decide it got the lab wrong too. Hub election by address prefix failed silently when nobody knew the convention. No private key travels, and that is the whole design. The node generated its own keypair and kept the private half; the configuration points at a file the node wrote, using WireGuard's own PostUp. So the control plane composes a complete configuration for a node it cannot pretend to be -- it knows every public key and holds none of the private ones. Delivered as an ordinary declaration: a package, a file and a service. The host does not know what a private network is and does not learn one. There is a test holding that line, because the moment connectivity needs a new shape in tier 0 is the moment the host stops being small enough to trust. The generated file is written to be read: each peer says why it is there, a peer with no endpoint says why it has none, and the header says not to edit it -- an edit survives until the graph next changes and then vanishes, which is worse than never being applied, because the machine works and then stops and nothing changed that anybody remembers. Fault injection found one weak test. The keepalive rule was asserted only against the hub, whose peer entries happen not to set the field at all, so it was testing an absence rather than the rule. It now checks two direct peers where one is reachable and one is not.
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
-- What the mesh needs in order to compute a private network.
|
||||
--
|
||||
-- novox/hq 08-connectivity. Three declared inputs and one reported key. All four are facts about
|
||||
-- a node that only the mesh can hold, because computing the graph needs every node at once —
|
||||
-- which is the definition of control-plane work.
|
||||
|
||||
-- The node's public key on the overlay. Reported by the node, which generated the pair and kept
|
||||
-- the private half. So the control plane computes a graph it cannot itself impersonate.
|
||||
alter table node add column overlay_key text;
|
||||
|
||||
-- Where the node can be dialled, or null for nowhere.
|
||||
--
|
||||
-- DECLARED, never inferred from the address. The address is evidence of reachability and is not
|
||||
-- the fact: carrier-grade NAT looks public and is not, a routable address behind a closed
|
||||
-- firewall looks public and is not, and the regular expression that used to decide this got the
|
||||
-- lab wrong as well (novox/hq ADR 0007).
|
||||
alter table node add column endpoint text;
|
||||
|
||||
-- Where the machine physically is, or null if it roams.
|
||||
--
|
||||
-- Two nodes at one site peer directly; everything else routes through the hub. A node with no
|
||||
-- site is hub-only, and that is not a simplification — WireGuard has no failover, so a more
|
||||
-- specific route to a dead endpoint blackholes rather than falling back. One path is better than
|
||||
-- two when one of them can swallow traffic silently.
|
||||
alter table node add column site text;
|
||||
|
||||
-- Whether this node is the hub. DECLARED, never derived from an address prefix: an election
|
||||
-- decided by the first four characters of an address fails silently, cannot be queried, and makes
|
||||
-- renumbering an outage.
|
||||
alter table node add column is_hub boolean not null default false;
|
||||
|
||||
-- At most one hub. Partial, so it constrains the true ones and says nothing about the rest.
|
||||
create unique index node_one_hub on node ((is_hub)) where is_hub;
|
||||
|
||||
-- The node's address on the overlay, assigned by the mesh. A node computes nothing about the
|
||||
-- network it is joining: it generates a keypair, publishes the public half, and receives the rest.
|
||||
alter table node add column overlay_address inet;
|
||||
create unique index node_overlay_address on node (overlay_address) where overlay_address is not null;
|
||||
@@ -283,3 +283,80 @@ func (i *Inventory) Owned(ctx context.Context, node string) ([]string, time.Time
|
||||
}
|
||||
return owned, *reported, nil
|
||||
}
|
||||
|
||||
// Overlay is what the mesh knows about one node's place on the private network.
|
||||
type Overlay struct {
|
||||
Node string
|
||||
Name string
|
||||
Key string
|
||||
Endpoint string
|
||||
Site string
|
||||
Hub bool
|
||||
Address string
|
||||
}
|
||||
|
||||
// Reachable reports whether other nodes can dial this one.
|
||||
//
|
||||
// From the endpoint alone, which is declared. Never from the shape of an address: that inference
|
||||
// is wrong for carrier-grade NAT, wrong for IPv6, and wrong for a routable address behind a
|
||||
// closed firewall (novox/hq ADR 0007).
|
||||
func (o Overlay) Reachable() bool { return strings.TrimSpace(o.Endpoint) != "" }
|
||||
|
||||
// RecordOverlayKey keeps the public half a node generated.
|
||||
func (i *Inventory) RecordOverlayKey(ctx context.Context, node, key string) error {
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return errors.New("a node reported an empty overlay key")
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set overlay_key = $2 where id = $1`, node, key)
|
||||
return err
|
||||
}
|
||||
|
||||
// Overlays is every node's place on the private network, which is what computing the graph needs.
|
||||
//
|
||||
// Every node at once, deliberately: a peer list is derived from all of them, and that is the
|
||||
// whole reason this is the control plane's work rather than a node's.
|
||||
func (i *Inventory) Overlays(ctx context.Context) ([]Overlay, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, name, coalesce(overlay_key,''), coalesce(endpoint,''), coalesce(site,''),
|
||||
is_hub, coalesce(host(overlay_address),'')
|
||||
from node order by name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Overlay
|
||||
for rows.Next() {
|
||||
var o Overlay
|
||||
if err := rows.Scan(&o.Node, &o.Name, &o.Key, &o.Endpoint, &o.Site, &o.Hub, &o.Address); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ErrNotOneHub is what the mesh says when the graph cannot be computed.
|
||||
//
|
||||
// Its own error because it is not a fault in any node: it means nobody has said which node is the
|
||||
// hub, and a mesh with no hub has no path between sites at all. The old arrangement inferred this
|
||||
// from an address prefix and failed silently when nobody knew the convention.
|
||||
var ErrNotOneHub = errors.New("this mesh has no hub, so there is no path between sites")
|
||||
|
||||
// SetPlace declares where a node is and how it is reached.
|
||||
func (i *Inventory) SetPlace(ctx context.Context, name, endpoint, site string, hub bool, address string) error {
|
||||
node, err := i.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var addr any
|
||||
if strings.TrimSpace(address) != "" {
|
||||
addr = address
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set endpoint = nullif($2,''), site = nullif($3,''), is_hub = $4,
|
||||
overlay_address = $5::inet
|
||||
where id = $1`, node.ID, endpoint, site, hub, addr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The overlay is delivered as an ordinary declaration.
|
||||
//
|
||||
// novox/hq 08-connectivity: what the host receives is an interface configuration and a peer list,
|
||||
// as files. It does not compute them and it does not know what a private network is — the shapes
|
||||
// it already has are enough, which is why connectivity needs nothing new from tier 0.
|
||||
//
|
||||
// **No private key travels.** The configuration points at a file the node wrote from a key the
|
||||
// mesh has never seen, using WireGuard's own ability to set one after the interface is up. So the
|
||||
// control plane composes a complete configuration for a node it cannot impersonate.
|
||||
|
||||
// Interface is what the private network is called on a machine, and where the node's own key
|
||||
// lives. A constant rather than a setting: two nodes disagreeing about the name would produce a
|
||||
// mesh where each is configured correctly and nothing meets.
|
||||
const (
|
||||
Interface = "mesh0"
|
||||
ConfigPath = "/etc/wireguard/" + Interface + ".conf"
|
||||
Unit = "wg-quick@" + Interface
|
||||
DefaultKeyPath = "/var/lib/mesh-host/overlay.key"
|
||||
)
|
||||
|
||||
// Resource is one entry in a declaration, built here and read by the host.
|
||||
type Resource map[string]any
|
||||
|
||||
// Declaration is what the mesh sends a node to put it on the network.
|
||||
//
|
||||
// Three resources and nothing clever: the tools, the configuration, and the interface running. A
|
||||
// person can read it, which is the point — this is the first thing a node is ever told, and if it
|
||||
// is wrong the node is unreachable and the mistake has to be findable by eye.
|
||||
func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
|
||||
if node.Address == "" {
|
||||
return nil, fmt.Errorf("%s has no address on the overlay, so there is nothing to configure",
|
||||
node.Name)
|
||||
}
|
||||
if keyPath == "" {
|
||||
keyPath = DefaultKeyPath
|
||||
}
|
||||
|
||||
resources := []Resource{
|
||||
{
|
||||
"id": "overlay-tools", "type": "package", "package": "wireguard-tools",
|
||||
},
|
||||
{
|
||||
"id": "overlay-config", "type": "file", "path": ConfigPath,
|
||||
// Readable only by root: it lists every peer's key and endpoint, which is a map of
|
||||
// the mesh. Not secret in the way a private key is, and not something to leave
|
||||
// world-readable on a machine somebody else also uses.
|
||||
"mode": "0600",
|
||||
"content": config(node, peers, keyPath),
|
||||
},
|
||||
{
|
||||
"id": "overlay-up", "type": "service", "unit": Unit,
|
||||
"state": "running",
|
||||
// Enabled, so the node comes back onto the network after a reboot without waiting to
|
||||
// be told again. A node whose overlay only exists while something is watching is not
|
||||
// a node that survives being switched off and on.
|
||||
"boot": "enabled",
|
||||
},
|
||||
}
|
||||
|
||||
return json.Marshal(map[string]any{"declaration": 1, "resources": resources})
|
||||
}
|
||||
|
||||
// config writes the interface file.
|
||||
//
|
||||
// Deliberately in the order a person would read it: who I am, then who I talk to, each with a
|
||||
// line saying why it is there. A generated file that cannot be understood by the person it
|
||||
// confuses is a generated file that gets edited by hand.
|
||||
func config(node Node, peers []Peer, keyPath string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever the\n")
|
||||
b.WriteString("# peer graph changes, and an edit would survive until the next change and\n")
|
||||
b.WriteString("# then vanish, which is worse than not being applied at all.\n")
|
||||
fmt.Fprintf(&b, "#\n# node %s", node.Name)
|
||||
if node.Site != "" {
|
||||
fmt.Fprintf(&b, ", at %s", node.Site)
|
||||
}
|
||||
if !node.Reachable() {
|
||||
b.WriteString(", not dialable — it opens every path itself")
|
||||
}
|
||||
b.WriteString("\n\n[Interface]\n")
|
||||
fmt.Fprintf(&b, "Address = %s/32\n", node.Address)
|
||||
if node.Reachable() {
|
||||
if port := portOf(node.Endpoint); port != "" {
|
||||
fmt.Fprintf(&b, "ListenPort = %s\n", port)
|
||||
}
|
||||
}
|
||||
// The private key is set from a file the node wrote, so it never appears here and never
|
||||
// travelled. Everything else in this file came from the mesh; this one line is the node's.
|
||||
fmt.Fprintf(&b, "PostUp = wg set %%i private-key %s\n", keyPath)
|
||||
|
||||
for _, p := range peers {
|
||||
fmt.Fprintf(&b, "\n# %s — %s\n[Peer]\n", p.Name, p.Why)
|
||||
fmt.Fprintf(&b, "PublicKey = %s\n", p.Key)
|
||||
fmt.Fprintf(&b, "AllowedIPs = %s\n", p.Allowed)
|
||||
if p.Endpoint != "" {
|
||||
fmt.Fprintf(&b, "Endpoint = %s\n", p.Endpoint)
|
||||
} else {
|
||||
b.WriteString("# no endpoint: this peer cannot be dialled and opens the path itself\n")
|
||||
}
|
||||
if p.Keepalive {
|
||||
b.WriteString("PersistentKeepalive = 25\n")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func portOf(endpoint string) string {
|
||||
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||
return endpoint[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func declarationFor(t *testing.T, node Node, peers []Peer) (string, []map[string]any) {
|
||||
t.Helper()
|
||||
raw, err := Declaration(node, peers, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var d struct {
|
||||
Declaration int `json:"declaration"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Declaration != 1 {
|
||||
t.Fatalf("declaration version %d", d.Declaration)
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
if r["type"] == "file" {
|
||||
return r["content"].(string), d.Resources
|
||||
}
|
||||
}
|
||||
t.Fatal("the declaration has no configuration file in it")
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func TestNoPrivateKeyEverTravels(t *testing.T) {
|
||||
// The property the whole design rests on: the node generated its keypair and kept the private
|
||||
// half, so the mesh composes a configuration for a node it cannot impersonate. A private key
|
||||
// appearing here would mean the control plane had one — and a copy of its database would then
|
||||
// be every node's network identity.
|
||||
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"},
|
||||
[]Peer{{Name: "anchor", Key: "HUB", Allowed: "10.42.0.0/16", Endpoint: "198.51.100.1:51820"}})
|
||||
|
||||
if strings.Contains(config, "PrivateKey") {
|
||||
t.Error("the configuration carries a PrivateKey line; the mesh must never hold one")
|
||||
}
|
||||
if !strings.Contains(config, "private-key "+DefaultKeyPath) {
|
||||
t.Error("the configuration does not point at the key file the node wrote, so the " +
|
||||
"interface would come up with no key at all")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDeclarationUsesOnlyShapesTheHostAlreadyHas(t *testing.T) {
|
||||
// Connectivity needs nothing new from tier 0, and that is worth holding: the host does not
|
||||
// know what a private network is, and should not learn.
|
||||
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
|
||||
|
||||
allowed := map[string]bool{"package": true, "file": true, "service": true,
|
||||
"directory": true, "container": true, "action": true}
|
||||
for _, r := range resources {
|
||||
if !allowed[r["type"].(string)] {
|
||||
t.Errorf("the overlay declaration uses %q, which the host does not have", r["type"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheInterfaceComesBackAfterAReboot(t *testing.T) {
|
||||
// A node whose overlay only exists while something is watching is not a node that survives
|
||||
// being switched off and on — and it would come back unreachable, which is the worst way to
|
||||
// come back.
|
||||
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
|
||||
for _, r := range resources {
|
||||
if r["type"] == "service" {
|
||||
if r["boot"] != "enabled" {
|
||||
t.Error("the overlay interface is not enabled at boot")
|
||||
}
|
||||
if r["state"] != "running" {
|
||||
t.Error("the overlay interface is not asked to be running")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("nothing in the declaration brings the interface up")
|
||||
}
|
||||
|
||||
func TestTheConfigurationIsNotWorldReadable(t *testing.T) {
|
||||
// It lists every peer's key and endpoint, which is a map of the mesh. Not secret the way a
|
||||
// private key is, and not something to leave readable on a machine somebody else also uses.
|
||||
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
|
||||
for _, r := range resources {
|
||||
if r["type"] == "file" && r["mode"] != "0600" {
|
||||
t.Errorf("the peer list is mode %v", r["mode"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPeerThatCannotBeDialledSaysSo(t *testing.T) {
|
||||
// A [Peer] with no Endpoint is correct and looks like a mistake. Saying why stops somebody
|
||||
// helpfully adding one that cannot work.
|
||||
config, _ := declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
|
||||
Endpoint: "198.51.100.1:51820", Hub: true},
|
||||
[]Peer{{Name: "laptop", Key: "PUB", Allowed: "10.42.0.2/32", Why: "routes through this hub"}})
|
||||
|
||||
if strings.Contains(config, "Endpoint =") {
|
||||
t.Error("an endpoint was written for a peer that has none")
|
||||
}
|
||||
if !strings.Contains(config, "cannot be dialled") {
|
||||
t.Error("the file does not say why that peer has no endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFileSaysNotToEditIt(t *testing.T) {
|
||||
// It is replaced whenever the graph changes. An edit survives until then and vanishes, which
|
||||
// is worse than never being applied — the machine works, then stops, and nothing changed
|
||||
// that anybody remembers.
|
||||
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
|
||||
if !strings.Contains(config, "Do not edit") {
|
||||
t.Error("a generated file does not say it is generated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithNoAddressIsRefused(t *testing.T) {
|
||||
// Rather than a configuration with a blank address, which wg-quick would reject on the
|
||||
// machine, at boot, where the failure is much harder to see.
|
||||
if _, err := Declaration(Node{Name: "laptop", Key: "PUB"}, nil, ""); err == nil {
|
||||
t.Fatal("a node with no overlay address was given a configuration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyAReachableNodeListens(t *testing.T) {
|
||||
// A ListenPort on a node nothing can dial is a port open for no reason.
|
||||
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
|
||||
if strings.Contains(config, "ListenPort") {
|
||||
t.Error("a node that cannot be dialled was told to listen")
|
||||
}
|
||||
config, _ = declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
|
||||
Endpoint: "198.51.100.1:51820"}, nil)
|
||||
if !strings.Contains(config, "ListenPort = 51820") {
|
||||
t.Error("a reachable node does not listen on the port its endpoint names")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package overlay computes the private network every node runs on.
|
||||
//
|
||||
// novox/hq 08-connectivity. This is control-plane work by definition: a peer list is derived from
|
||||
// every node at once, and no node has that. A node computes nothing about the mesh — it generates
|
||||
// a keypair, publishes the public half, and receives the rest.
|
||||
//
|
||||
// The shape is a hub, with direct peering between nodes at the same site. Not a full mesh, and
|
||||
// the reason is a property of WireGuard rather than a preference: there is no failover. A more
|
||||
// specific route to a dead endpoint blackholes; it does not fall back to the general one. So a
|
||||
// node gets exactly one path to any peer, because two would mean one of them silently swallowing
|
||||
// traffic.
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Node is one machine's place on the network, as the mesh holds it.
|
||||
type Node struct {
|
||||
Name string
|
||||
Key string
|
||||
Endpoint string
|
||||
Site string
|
||||
Hub bool
|
||||
Address string
|
||||
}
|
||||
|
||||
// Reachable reports whether other nodes can dial this one. Declared, never inferred.
|
||||
func (n Node) Reachable() bool { return strings.TrimSpace(n.Endpoint) != "" }
|
||||
|
||||
// Peer is one entry in a node's peer list.
|
||||
type Peer struct {
|
||||
Name string
|
||||
Key string
|
||||
// Endpoint is empty when this peer cannot be dialled — it must dial us instead.
|
||||
Endpoint string
|
||||
// Allowed is what traffic goes down this tunnel. A single address for a direct peer; the
|
||||
// whole overlay for the hub, which is what makes it the route of last resort.
|
||||
Allowed string
|
||||
// Keepalive matters only on the side behind NAT: a node that cannot be dialled has to keep
|
||||
// the path open from its end, or the peer's first packet arrives at a mapping that has
|
||||
// already expired.
|
||||
Keepalive bool
|
||||
// Why this peer is in the list, for a person reading a generated file and wondering.
|
||||
Why string
|
||||
}
|
||||
|
||||
// Graph is every node's peer list.
|
||||
type Graph map[string][]Peer
|
||||
|
||||
// ErrNoHub means nobody has said which node is the hub.
|
||||
//
|
||||
// Its own error rather than an empty graph: a mesh with no hub has no path between sites, and
|
||||
// answering with "no peers" would look like a working mesh where nothing can reach anything.
|
||||
var ErrNoHub = errors.New("this mesh has no hub, so there is no path between sites")
|
||||
|
||||
// Compute derives every node's peer list.
|
||||
//
|
||||
// Nodes without a key or an address are skipped rather than refused: a node that has enrolled and
|
||||
// not yet been given a place on the network is an ordinary in-between state, and failing the whole
|
||||
// graph because one node is half-configured would mean no node gets a network.
|
||||
func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
|
||||
var hub *Node
|
||||
usable := make([]Node, 0, len(nodes))
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
if n.Key == "" || n.Address == "" {
|
||||
continue
|
||||
}
|
||||
usable = append(usable, n)
|
||||
if n.Hub {
|
||||
hub = &usable[len(usable)-1]
|
||||
}
|
||||
}
|
||||
if len(usable) == 0 {
|
||||
return Graph{}, nil
|
||||
}
|
||||
if hub == nil {
|
||||
return nil, ErrNoHub
|
||||
}
|
||||
if !hub.Reachable() {
|
||||
return nil, fmt.Errorf(
|
||||
"%s is the hub and has no endpoint, so nothing can dial it. The hub is the one node "+
|
||||
"that must be reachable from wherever the others are", hub.Name)
|
||||
}
|
||||
|
||||
graph := Graph{}
|
||||
for _, self := range usable {
|
||||
var peers []Peer
|
||||
|
||||
for _, other := range usable {
|
||||
if other.Name == self.Name {
|
||||
continue
|
||||
}
|
||||
// Two nodes at the same site peer directly. A site is where a machine physically is,
|
||||
// and machines that share one have a path that does not need the hub — so using it
|
||||
// keeps their traffic off a link that may be somewhere else entirely.
|
||||
if self.Site != "" && self.Site == other.Site {
|
||||
peers = append(peers, Peer{
|
||||
Name: other.Name, Key: other.Key,
|
||||
Endpoint: other.Endpoint,
|
||||
Allowed: other.Address + "/32",
|
||||
Keepalive: !self.Reachable(),
|
||||
Why: "at the same site",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !self.Hub {
|
||||
// Everything else goes through the hub, including a node that roams. AllowedIPs is
|
||||
// the whole overlay, so this is the route of last resort — and because direct peers
|
||||
// above are single addresses, they win on specificity without either being ambiguous.
|
||||
peers = append(peers, Peer{
|
||||
Name: hub.Name, Key: hub.Key,
|
||||
Endpoint: hub.Endpoint,
|
||||
Allowed: overlayCIDR,
|
||||
Keepalive: !self.Reachable(),
|
||||
Why: "the hub — everything not at this site",
|
||||
})
|
||||
} else {
|
||||
// The hub holds every node that does not share a site with it, because those nodes
|
||||
// route through it and it must know where to send the replies. Ones it cannot dial
|
||||
// will dial it.
|
||||
for _, other := range usable {
|
||||
if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
|
||||
continue
|
||||
}
|
||||
peers = append(peers, Peer{
|
||||
Name: other.Name, Key: other.Key,
|
||||
Endpoint: other.Endpoint,
|
||||
Allowed: other.Address + "/32",
|
||||
Why: "routes through this hub",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
|
||||
graph[self.Name] = peers
|
||||
}
|
||||
return graph, nil
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package overlay
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const cidr = "10.42.0.0/16"
|
||||
|
||||
func at(name, site, address, endpoint string, hub bool) Node {
|
||||
return Node{Name: name, Key: "key-" + name, Site: site, Address: address,
|
||||
Endpoint: endpoint, Hub: hub}
|
||||
}
|
||||
|
||||
func peersOf(t *testing.T, g Graph, node string) map[string]Peer {
|
||||
t.Helper()
|
||||
out := map[string]Peer{}
|
||||
for _, p := range g[node] {
|
||||
out[p.Name] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestEveryNodeReachesTheHub(t *testing.T) {
|
||||
// The property that makes this a network at all. Without it a node has no route to anything
|
||||
// it does not share a site with.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
at("home", "house", "10.42.0.3", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"laptop", "home"} {
|
||||
hub, ok := peersOf(t, g, node)["anchor"]
|
||||
if !ok {
|
||||
t.Fatalf("%s has no route to the hub", node)
|
||||
}
|
||||
if hub.Allowed != cidr {
|
||||
t.Errorf("%s routes %s through the hub; it must be the whole overlay or the hub is "+
|
||||
"not a route of last resort", node, hub.Allowed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesAtOneSitePeerDirectly(t *testing.T) {
|
||||
// Machines that share a site have a path that does not need the hub, and using it keeps their
|
||||
// traffic off a link that may be on another continent.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
|
||||
at("server", "house", "10.42.0.3", "192.0.2.3:51820", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
direct, ok := peersOf(t, g, "desk")["server"]
|
||||
if !ok {
|
||||
t.Fatal("two machines at one site do not peer directly")
|
||||
}
|
||||
if direct.Allowed != "10.42.0.3/32" {
|
||||
t.Errorf("the direct peer allows %s; a single address is what makes it win on "+
|
||||
"specificity over the hub's whole-overlay route", direct.Allowed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARoamingNodeGetsExactlyOnePath(t *testing.T) {
|
||||
// Hub-only, and not as a simplification. WireGuard has no failover: a more specific route to
|
||||
// a dead endpoint blackholes rather than falling back, so two paths would mean one of them
|
||||
// silently swallowing traffic.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
at("other", "", "10.42.0.3", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(g["laptop"]); got != 1 {
|
||||
t.Fatalf("a roaming node has %d peers; it gets exactly one path", got)
|
||||
}
|
||||
if g["laptop"][0].Name != "anchor" {
|
||||
t.Errorf("a roaming node's single path is %s, not the hub", g["laptop"][0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoRoamingNodesDoNotPeerWithEachOther(t *testing.T) {
|
||||
// An empty site is not a site. Nodes that roam have no shared location, and treating "" as
|
||||
// one would have every roaming machine try to dial every other, none of which can be dialled.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
at("phone", "", "10.42.0.3", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := peersOf(t, g, "laptop")["phone"]; ok {
|
||||
t.Error("two nodes with no site peered as though they shared one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheSideThatCannotBeDialledKeepsThePathOpen(t *testing.T) {
|
||||
// Keepalive matters exactly once: on the node behind NAT. Without it the peer's first packet
|
||||
// arrives at a mapping that has already expired. On the reachable side it is pointless
|
||||
// traffic for ever.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !peersOf(t, g, "laptop")["anchor"].Keepalive {
|
||||
t.Error("a node that cannot be dialled does not keep its path open")
|
||||
}
|
||||
if peersOf(t, g, "anchor")["laptop"].Keepalive {
|
||||
t.Error("a reachable node sends keepalives it does not need")
|
||||
}
|
||||
|
||||
// And between two direct peers at one site, where whether to keep the path open depends on
|
||||
// which end you are. Checked here because the hub's own peer list happens not to set the
|
||||
// field at all, so asserting only against the hub tests an absence rather than the rule.
|
||||
same, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
|
||||
at("server", "house", "10.42.0.3", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !peersOf(t, same, "server")["desk"].Keepalive {
|
||||
t.Error("the peer that cannot be dialled does not keep the path open")
|
||||
}
|
||||
if peersOf(t, same, "desk")["server"].Keepalive {
|
||||
t.Error("the peer that can be dialled sends keepalives it does not need")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHubKnowsEveryoneItRoutesFor(t *testing.T) {
|
||||
// The replies have to get back. A hub that does not hold a peer cannot answer it.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
at("home", "house", "10.42.0.3", "", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub := peersOf(t, g, "anchor")
|
||||
for _, want := range []string{"laptop", "home"} {
|
||||
if _, ok := hub[want]; !ok {
|
||||
t.Errorf("the hub does not hold %s, so replies to it have nowhere to go", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeshWithNoHubIsRefused(t *testing.T) {
|
||||
// Not an empty graph. A mesh with no hub has no path between sites, and answering "no peers"
|
||||
// would look like a working network in which nothing can reach anything — which is how the
|
||||
// old arrangement failed, silently, when nobody knew the address convention.
|
||||
_, err := Compute([]Node{
|
||||
at("a", "", "10.42.0.1", "", false),
|
||||
at("b", "", "10.42.0.2", "", false),
|
||||
}, cidr)
|
||||
if !errors.Is(err, ErrNoHub) {
|
||||
t.Fatalf("a mesh with no hub gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreachableHubIsRefused(t *testing.T) {
|
||||
// The hub is the one node that must be dialable from wherever the others are. Left
|
||||
// unchecked, every node would be given a peer it can never reach and the mesh would look
|
||||
// configured and be silent.
|
||||
_, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
}, cidr)
|
||||
if err == nil {
|
||||
t.Fatal("a hub with no endpoint was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must be reachable") {
|
||||
t.Errorf("the refusal does not say why: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithNoPlaceYetIsSkippedRatherThanFatal(t *testing.T) {
|
||||
// A node that has enrolled and not yet been given an address is an ordinary in-between state.
|
||||
// Failing the whole graph over it would mean no node gets a network because one is half done.
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("laptop", "", "10.42.0.2", "", false),
|
||||
{Name: "newcomer", Key: "", Address: ""},
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := g["newcomer"]; ok {
|
||||
t.Error("a node with no key or address was given a peer list")
|
||||
}
|
||||
if _, ok := peersOf(t, g, "laptop")["newcomer"]; ok {
|
||||
t.Error("a node with no key was put in somebody's peer list")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNobodyPeersWithThemselves(t *testing.T) {
|
||||
g, err := Compute([]Node{
|
||||
at("anchor", "house", "10.42.0.1", "198.51.100.1:51820", true),
|
||||
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
|
||||
}, cidr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for node, peers := range g {
|
||||
for _, p := range peers {
|
||||
if p.Name == node {
|
||||
t.Errorf("%s peers with itself", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user