diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index 9a794fc..d6b2580 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -13,6 +13,7 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" "time" @@ -20,6 +21,7 @@ import ( "github.com/novox/mesh-control/internal/identity" "github.com/novox/mesh-control/internal/inventory" "github.com/novox/mesh-control/internal/link" + "github.com/novox/mesh-control/internal/overlay" "github.com/novox/mesh-control/internal/store" "github.com/novox/mesh-control/internal/token" ) @@ -72,6 +74,8 @@ func run() error { return serve(ctx) case "declare": return declare(ctx, args[1:]) + case "overlay": + return overlayCommand(ctx, args[1:]) case "version": fmt.Println(version) return nil @@ -96,6 +100,9 @@ func usage() { broker show where the broker is, and what to expect there serve consume what nodes say, and answer declare send a node a signed declaration + overlay place [flags] say where a node is and how it is reached + overlay show the private network, as the mesh computes it + overlay push send every node its part of the private network version what this binary is Each context reaches its own store through its own credential (novox/hq ADR 0008), named @@ -459,3 +466,170 @@ func declare(ctx context.Context, args []string) error { fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) return nil } + +// OverlayCIDRVar is the range the mesh allocates node addresses from. +const OverlayCIDRVar = "MESH_OVERLAY_CIDR" + +func overlayCIDR() string { + if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" { + return v + } + return "10.42.0.0/16" +} + +func overlayCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("overlay place [flags], overlay show, or overlay push") + } + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + switch args[0] { + case "place": + return overlayPlace(ctx, inv, args[1:]) + case "show": + return overlayShow(ctx, inv) + case "push": + return overlayPush(ctx, inv) + default: + return fmt.Errorf("overlay has no %q; it has place, show and push", args[0]) + } +} + +func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error { + if len(args) == 0 { + return errors.New("overlay place [--endpoint host:port] [--site name] [--hub]") + } + node := args[0] + + set := flag.NewFlagSet("overlay place", flag.ContinueOnError) + endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere") + site := set.String("site", "", "where this machine physically is, or empty if it roams") + hub := set.Bool("hub", false, "this node is the hub every other routes through") + if err := set.Parse(args[1:]); err != nil { + return err + } + + // Declared, all three. The address is evidence of reachability and is not the fact, and hub + // election by address prefix fails silently (novox/hq ADR 0007). + if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil { + return err + } + found, err := inv.NodeByName(ctx, node) + if err != nil { + return err + } + address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR()) + if err != nil { + return err + } + + fmt.Printf("%s is at %s on the overlay\n", node, address) + switch { + case *hub: + fmt.Println(" the hub — every node not sharing a site routes through it") + case *endpoint == "": + fmt.Println(" not dialable — it opens every path itself") + } + if *site != "" { + fmt.Printf(" at %s, so it peers directly with anything else there\n", *site) + } + return nil +} + +// graph reads every node's place and computes the network. Every node at once, which is the whole +// reason this is the control plane's work. +func graph(ctx context.Context, inv *inventory.Inventory) ([]overlay.Node, overlay.Graph, error) { + places, err := inv.Overlays(ctx) + if err != nil { + return nil, nil, err + } + nodes := make([]overlay.Node, 0, len(places)) + for _, p := range places { + nodes = append(nodes, overlay.Node{ + Name: p.Name, Key: p.Key, Endpoint: p.Endpoint, + Site: p.Site, Hub: p.Hub, Address: p.Address, + }) + } + computed, err := overlay.Compute(nodes, overlayCIDR()) + return nodes, computed, err +} + +func overlayShow(ctx context.Context, inv *inventory.Inventory) error { + nodes, computed, err := graph(ctx, inv) + if err != nil { + return err + } + if len(nodes) == 0 { + fmt.Println("this mesh has no nodes") + return nil + } + + for _, n := range nodes { + place := n.Address + if place == "" { + // Said, not skipped. A node with no place is a node with no network, and it should + // be visible here rather than quietly absent from a list of who is on it. + place = "no address — run `overlay place`" + } + fmt.Printf("%-16s %-14s", n.Name, place) + switch { + case n.Hub: + fmt.Print(" hub") + case !n.Reachable(): + fmt.Print(" not dialable") + } + if n.Site != "" { + fmt.Printf(" at %s", n.Site) + } + fmt.Println() + for _, p := range computed[n.Name] { + fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why) + } + } + return nil +} + +func overlayPush(ctx context.Context, inv *inventory.Inventory) error { + nodes, computed, err := graph(ctx, inv) + if err != nil { + return err + } + + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + sent := 0 + for _, n := range nodes { + peers, ok := computed[n.Name] + if !ok { + // Skipped, and said. A node with no key or address is not on the network yet, and + // sending it an empty configuration would take down the one it may already have. + fmt.Printf("%s is not on the overlay yet — skipped\n", n.Name) + continue + } + declaration, err := overlay.Declaration(n, peers, "") + if err != nil { + return err + } + if err := link.Declare(ctx, server.Channel(), ident, n.Name, declaration, 15*time.Second); err != nil { + return err + } + fmt.Printf("sent %s its place on the overlay — %d peer(s)\n", n.Name, len(peers)) + sent++ + } + fmt.Printf("\n%d of %d node(s) told\n", sent, len(nodes)) + return nil +} diff --git a/internal/inventory/addresses.go b/internal/inventory/addresses.go new file mode 100644 index 0000000..3826e0b --- /dev/null +++ b/internal/inventory/addresses.go @@ -0,0 +1,76 @@ +package inventory + +import ( + "context" + "fmt" + "net/netip" +) + +// The mesh assigns overlay addresses. A node computes nothing about the network it is joining — +// it generates a keypair, publishes the public half, and receives the rest +// (novox/hq 08-connectivity). + +// AssignAddress gives a node its place on the private network, if it has none. +// +// Idempotent: a node that already has an address keeps it. Reassigning would change every other +// node's peer list to chase it, and an address that moves is the thing declaring the hub was +// meant to stop. +// +// Allocated in order from the range, taking the lowest free one. Not random: a person reading a +// peer list should be able to guess which node an address belongs to, and reuse of a released +// address is a smaller problem than a list nobody can hold in their head. +func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) { + prefix, err := netip.ParsePrefix(cidr) + if err != nil { + return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err) + } + + var existing *string + if err := i.store.Pool().QueryRow(ctx, + `select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil { + return "", err + } + if existing != nil && *existing != "" { + return *existing, nil + } + + taken := map[string]bool{} + rows, err := i.store.Pool().Query(ctx, + `select host(overlay_address) from node where overlay_address is not null`) + if err != nil { + return "", err + } + for rows.Next() { + var a string + if err := rows.Scan(&a); err != nil { + rows.Close() + return "", err + } + taken[a] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return "", err + } + + // The first address in a range is conventionally the network itself and is skipped, so + // allocation starts one past it. + candidate := prefix.Masked().Addr().Next() + for prefix.Contains(candidate) { + if !taken[candidate.String()] { + if _, err := i.store.Pool().Exec(ctx, + `update node set overlay_address = $2::inet where id = $1`, + node, candidate.String()); err != nil { + return "", err + } + return candidate.String(), nil + } + candidate = candidate.Next() + } + // Said plainly rather than returning an empty address that fails later on a machine. A mesh + // that has outgrown its range needs a person, and renumbering is not something to attempt + // halfway through assigning one node. + return "", fmt.Errorf( + "every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+ + "range and renumbering it is a deliberate act", cidr, node) +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 480e7be..df52638 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -32,8 +32,8 @@ type Enrolment struct { // single statement that both finds and marks it, so two machines racing on one secret produce one // winner. Only then is a key recorded — because recording a key for a node whose token turned out // to be spent would leave the mesh believing a machine that never had the right to join. -func (e Enrolment) Enrol(ctx context.Context, secret string, public ed25519.PublicKey, - profile map[string]any) (EnrolReply, error) { +func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) { + secret, public, profile := request.Secret, ed25519.PublicKey(request.PublicKey), request.Profile if len(public) != ed25519.PublicKeySize { return EnrolReply{}, fmt.Errorf("a node presented a %d-byte key, and an identity is %d", @@ -84,6 +84,17 @@ func (e Enrolment) Enrol(ctx context.Context, secret string, public ed25519.Publ reply.Password = password } + // Recorded before the profile because the overlay is the first declaration this node will + // receive, and without this key the mesh cannot compose one. A node enrolled with no overlay + // key is a node the graph skips — an ordinary in-between state, and one worth leaving as + // briefly as possible. + if request.OverlayKey != "" { + if err := e.Inventory.RecordOverlayKey(ctx, node.ID, request.OverlayKey); err != nil { + return EnrolReply{}, fmt.Errorf( + "the token was spent and %s's overlay key could not be recorded: %w", node.Name, err) + } + } + if profile != nil { // Not fatal if it fails. The profile is what the control plane needs in order to decide // what this machine should run, and it is reported again on every connection — so losing diff --git a/internal/link/protocol.go b/internal/link/protocol.go index f3f0f1b..e384592 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -40,6 +40,10 @@ type EnrolRequest struct { // half has never left that machine (novox/hq ADR 0004). PublicKey []byte `json:"public_key"` + // OverlayKey is the public half of this node's key on the private network — a different key + // from PublicKey, and the mesh only ever sees this half. + OverlayKey string `json:"overlay_key,omitempty"` + // Profile is what this machine can be asked to do. The control plane cannot decide what a // node should run without it, so it arrives with enrolment rather than being asked for after. Profile map[string]any `json:"profile,omitempty"` diff --git a/internal/link/serve.go b/internal/link/serve.go index c42b495..0a290e2 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -2,7 +2,6 @@ package link import ( "context" - "crypto/ed25519" "encoding/json" "errors" "fmt" @@ -24,7 +23,7 @@ const AMQPVar = "MESH_BROKER_AMQP" type Enroller interface { // Enrol spends the token, records the key, and reports the node's name. The error is // returned to the node as a refusal; it must be the same for every reason a token can fail. - Enrol(ctx context.Context, secret string, public ed25519.PublicKey, profile map[string]any) (EnrolReply, error) + Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) } // Server consumes what nodes say. @@ -194,7 +193,7 @@ func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) { if err := json.Unmarshal(delivery.Body, &request); err != nil { s.log.Printf("an enrolment request could not be read: %v", err) } else { - accepted, err := s.enroller.Enrol(ctx, request.Secret, request.PublicKey, request.Profile) + accepted, err := s.enroller.Enrol(ctx, request) if err != nil { // Logged in full here, where an operator can see it; sent back as one refusal, so // that somebody guessing learns nothing from which reason came back. diff --git a/internal/overlay/declaration.go b/internal/overlay/declaration.go index d3ec8fb..eef7aec 100644 --- a/internal/overlay/declaration.go +++ b/internal/overlay/declaration.go @@ -62,6 +62,19 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) { // be told again. A node whose overlay only exists while something is watching is not // a node that survives being switched off and on. "boot": "enabled", + // And restarted when the peer list changes, because a running interface does not + // re-read its configuration. + // + // This is the whole of it: a node joins, every existing node's peer list changes, + // each file is replaced — and without this the service is already running, nothing + // reloads it, and every node keeps a network that no longer matches the mesh. It + // reports complete success. The lab found it the moment a third node arrived. + // + // Declared state rather than a command. The service must reflect the file; the host + // works out that it does not. A command to restart would be an action, and the link + // may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly, + // which is how this shape was arrived at. + "restart-on": []string{"overlay-config"}, }, } @@ -96,6 +109,39 @@ func config(node Node, peers []Peer, keyPath string) string { // travelled. Everything else in this file came from the mesh; this one line is the node's. fmt.Fprintf(&b, "PostUp = wg set %%i private-key %s\n", keyPath) + if node.Hub { + // A hub carries traffic *between* its spokes, and a Linux machine does not forward + // packets unless it is told to. Without this every spoke reaches the hub perfectly and + // no spoke reaches any other — which is exactly how it failed in the lab, and it looks + // like a peering problem rather than a kernel setting. + // + // Here rather than in a separate resource because it is part of what being a hub means, + // and because it should last exactly as long as the interface does: a machine that stops + // being the hub should stop forwarding, and PostDown below is how that happens. + b.WriteString("PostUp = sysctl -q -w net.ipv4.ip_forward=1\n") + b.WriteString("PostDown = sysctl -q -w net.ipv4.ip_forward=0\n") + + // And past the machine's own firewall, which on any node with a container runtime is + // closed. Docker sets the FORWARD policy to DROP and inserts its chains, so the substrate + // this mesh installs at tier 1 silently breaks the network it builds at tier 2: every + // spoke reaches the hub, no spoke reaches any other, and every part of it reports + // success. Found in the lab; nothing about it is visible from the mesh's own state. + // + // Inserted at the top so it precedes those chains, and removed on the way down so a + // machine that stops being the hub stops carrying other people's traffic. Guarded on + // iptables existing: a machine without it has no policy to get past. + for _, direction := range []string{"-i", "-o"} { + fmt.Fprintf(&b, + "PostUp = command -v iptables >/dev/null && iptables -I FORWARD 1 %s %%i -j ACCEPT || true\n", + direction) + } + for _, direction := range []string{"-i", "-o"} { + fmt.Fprintf(&b, + "PostDown = command -v iptables >/dev/null && iptables -D FORWARD %s %%i -j ACCEPT || true\n", + direction) + } + } + for _, p := range peers { fmt.Fprintf(&b, "\n# %s — %s\n[Peer]\n", p.Name, p.Why) fmt.Fprintf(&b, "PublicKey = %s\n", p.Key) diff --git a/internal/overlay/declaration_test.go b/internal/overlay/declaration_test.go index ce64ebb..8e41884 100644 --- a/internal/overlay/declaration_test.go +++ b/internal/overlay/declaration_test.go @@ -2,6 +2,7 @@ package overlay import ( "encoding/json" + "fmt" "strings" "testing" ) @@ -137,3 +138,82 @@ func TestOnlyAReachableNodeListens(t *testing.T) { t.Error("a reachable node does not listen on the port its endpoint names") } } + +func TestTheServiceIsRestartedWhenThePeerListChanges(t *testing.T) { + // The fault this exists to catch, found in the lab the moment a third node arrived: a running + // WireGuard interface does not re-read its configuration. The file was replaced, the service + // was already running so nothing reloaded it, and every existing node kept a network that no + // longer matched the mesh — while looking entirely successful. + // + // So the declaration has to verify what the interface is *carrying*, not what the file says. + // + // And it must be declared state rather than a command: the host refuses an action arriving + // over the link (novox/hq ADR 0005), correctly, which is how this shape was arrived at. There + // is a test below that no action is ever in here. + _, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, + []Peer{{Name: "anchor", Key: "HUBKEY", Allowed: "10.42.0.0/16"}}) + + for _, r := range resources { + if r["type"] != "service" { + continue + } + reflects := fmt.Sprint(r["restart-on"]) + if !strings.Contains(reflects, "overlay-config") { + t.Errorf("the interface does not restart when its configuration changes: %v", reflects) + } + return + } + t.Error("nothing in the declaration brings the interface up") +} + +func TestTheOverlayDeclarationCarriesNoAction(t *testing.T) { + // The link may not carry an action, and the host refuses a declaration containing one — whole, + // not in part. An overlay declaration with an action in it does not half-apply: it leaves the + // node with no network at all. + _, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil) + for _, r := range resources { + if r["type"] == "action" { + t.Errorf("the declaration contains an action (%v); the host will refuse the whole "+ + "thing and the node will have no network", r["id"]) + } + } +} + +func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) { + // A hub carries traffic between its spokes, and Linux does not forward packets unless told + // to. Without it every spoke reaches the hub perfectly and no spoke reaches any other — which + // is how it failed in the lab, and it presents as a peering problem rather than a kernel + // setting, so it is worth being sure of. + hub, _ := declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1", + Endpoint: "198.51.100.1:51820", Hub: true}, nil) + if !strings.Contains(hub, "ip_forward=1") { + t.Error("the hub does not enable forwarding, so its spokes cannot reach each other") + } + if !strings.Contains(hub, "ip_forward=0") { + t.Error("the hub never stops forwarding; a machine that stops being the hub would keep " + + "passing traffic it is no longer part of") + } + + // And past the machine's own firewall. Any node with a container runtime has FORWARD set to + // DROP by Docker, so enabling ip_forward alone changes nothing — which is exactly how it + // failed, with every spoke reaching the hub and no spoke reaching any other. + // Checked as PostUp specifically. "contains FORWARD" passes on the PostDown line alone, + // which would leave a hub that tears down rules it never put up. + if !strings.Contains(hub, "PostUp = command -v iptables") { + t.Error("the hub does not open its own firewall, so a container runtime's DROP policy " + + "silently eats everything it was supposed to carry") + } + if !strings.Contains(hub, "PostDown = command -v iptables") { + t.Error("the hub never removes those rules, so a machine that stops being the hub keeps " + + "passing traffic it is no longer part of") + } + + spoke, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil) + if strings.Contains(spoke, "FORWARD") { + t.Error("a spoke was given forwarding rules it has no use for") + } + if strings.Contains(spoke, "ip_forward") { + t.Error("a spoke was told to forward packets, which is not its job and widens what a " + + "compromised one could do") + } +} diff --git a/internal/overlay/graph.go b/internal/overlay/graph.go index 692f5ab..0f1fa01 100644 --- a/internal/overlay/graph.go +++ b/internal/overlay/graph.go @@ -91,14 +91,28 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) { for _, self := range usable { var peers []Peer + // A node may share a site with the hub, and then the hub is one peer rather than two. + // Written before the loop because it changes what that loop may emit: WireGuard takes one + // entry per public key, so a hub appearing twice is a configuration it refuses — and the + // mesh would have produced it silently. The lab found this on the first two machines that + // shared a site with their hub. + hubIsHere := !self.Hub && self.Site != "" && self.Site == hub.Site + for _, other := range usable { - if other.Name == self.Name { + if other.Name == self.Name || (hubIsHere && other.Name == hub.Name) { continue } - // Two nodes at the same site peer directly. A site is where a machine physically is, - // and machines that share one have a path that does not need the hub — so using it - // keeps their traffic off a link that may be somewhere else entirely. - if self.Site != "" && self.Site == other.Site { + // Two nodes at the same site peer directly — but only if one of them can be + // dialled. If neither can, nobody opens the path, and the direct route is more + // specific than the hub's, so it wins and blackholes. That is this design's own + // stated hazard arriving in it: *a more specific route to a dead endpoint + // blackholes; it does not fall back to the general one.* + // + // Found in the lab with two machines at one site behind no reachable address, which + // is the ordinary shape of a home: they were given each other as peers, neither + // could start, and they could not reach each other at all while both reached the hub + // perfectly. + if self.Site != "" && self.Site == other.Site && (self.Reachable() || other.Reachable()) { peers = append(peers, Peer{ Name: other.Name, Key: other.Key, Endpoint: other.Endpoint, @@ -113,12 +127,19 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) { // Everything else goes through the hub, including a node that roams. AllowedIPs is // the whole overlay, so this is the route of last resort — and because direct peers // above are single addresses, they win on specificity without either being ambiguous. + why := "the hub — everything not at this site" + if hubIsHere { + // One entry doing both jobs: the direct path to a machine that happens to be + // here, and the route to everywhere else. Splitting them would need two entries + // for one key, which is the thing being avoided. + why = "the hub, which is also at this site — everything goes here" + } peers = append(peers, Peer{ Name: hub.Name, Key: hub.Key, Endpoint: hub.Endpoint, Allowed: overlayCIDR, Keepalive: !self.Reachable(), - Why: "the hub — everything not at this site", + Why: why, }) } else { // The hub holds every node that does not share a site with it, because those nodes diff --git a/internal/overlay/graph_test.go b/internal/overlay/graph_test.go index 3deb299..14b2592 100644 --- a/internal/overlay/graph_test.go +++ b/internal/overlay/graph_test.go @@ -222,3 +222,85 @@ func TestNobodyPeersWithThemselves(t *testing.T) { } } } + +func TestAHubAtYourOwnSiteAppearsOnceNotTwice(t *testing.T) { + // WireGuard takes one entry per public key. A hub that shares a site with a spoke was being + // emitted twice — once as a direct peer and once as the route of last resort — producing a + // configuration the interface refuses, from a mesh that thought it had succeeded. + // + // Found in the lab on the first two machines that shared a site with their hub, which is the + // ordinary case for a small mesh and was in none of the tests above. + g, err := Compute([]Node{ + at("anchor", "lab", "10.42.0.1", "192.0.2.10:51820", true), + at("laptop", "lab", "10.42.0.2", "", false), + }, cidr) + if err != nil { + t.Fatal(err) + } + + seen := map[string]int{} + for _, p := range g["laptop"] { + seen[p.Key]++ + } + for key, count := range seen { + if count > 1 { + t.Errorf("the key %s appears %d times; WireGuard takes one entry per key", key, count) + } + } + + if len(g["laptop"]) != 1 { + t.Fatalf("laptop has %d peers; the hub at its own site is one peer, not two", len(g["laptop"])) + } + // And that single entry has to carry everything, or the node keeps a direct path to the hub + // and loses its route to everywhere else. + if got := g["laptop"][0].Allowed; got != cidr { + t.Errorf("the single entry allows %s; it is both the direct path and the route of last "+ + "resort, so it carries the whole overlay", got) + } +} + +func TestTwoUnreachableNodesAtOneSiteDoNotPeerDirectly(t *testing.T) { + // Nobody would open the path, and the direct route is more specific than the hub's — so it + // wins and blackholes. Both nodes would reach the hub perfectly and be unable to reach each + // other, which is the worst arrangement available: it looks configured and is not. + // + // This is the design's own warning arriving in its implementation, and the lab found it with + // two machines at one site behind no reachable address — the ordinary shape of a house. + g, err := Compute([]Node{ + at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true), + at("desk", "house", "10.42.0.2", "", false), + at("server", "house", "10.42.0.3", "", false), + }, cidr) + if err != nil { + t.Fatal(err) + } + + if _, ok := peersOf(t, g, "desk")["server"]; ok { + t.Error("two nodes that neither can be dialled were peered directly; neither could open " + + "the path and the route is more specific than the hub's, so it blackholes") + } + // And they must still be able to reach each other — through the hub. + if hub := peersOf(t, g, "desk")["anchor"]; hub.Allowed != cidr { + t.Errorf("desk's route to everything else allows %s", hub.Allowed) + } +} + +func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) { + // The other side of it: if either end can be dialled, the path can be opened, and using it + // keeps their traffic off a hub that may be on another continent. + g, err := Compute([]Node{ + at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true), + at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false), + at("server", "house", "10.42.0.3", "", false), + }, cidr) + if err != nil { + t.Fatal(err) + } + if _, ok := peersOf(t, g, "server")["desk"]; !ok { + t.Error("a node did not peer with a reachable neighbour at its own site") + } + if _, ok := peersOf(t, g, "desk")["server"]; !ok { + t.Error("the reachable node did not hold its unreachable neighbour, so replies have " + + "nowhere to go") + } +}