diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 0f39abf..9941728 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -14,6 +14,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" ) // A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the @@ -45,6 +46,9 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node return nil } fmt.Printf(" firewall found %s\n", orNone(said.Firewall)) + if err := showTunnel(ctx, inv, node.Name); err != nil { + return err + } if len(said.Held) == 0 { fmt.Printf(" holding nothing found\n") } @@ -63,6 +67,44 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node return nil } +// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq +// ADR 0105): what it presented at enrolment, and what it last said about taking it over. +func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error { + tunnel, err := inv.TunnelOf(ctx, name) + if errors.Is(err, inventory.ErrNoTunnel) { + return nil + } + if err != nil { + return err + } + fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n", + tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers)) + carried, said, err := inv.CarriedTunnelOf(ctx, name) + if err != nil { + return err + } + switch { + case !said: + fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "") + case carried.State == inventory.CarriedTaken: + fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+ + "runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers) + case carried.State == inventory.CarriedDown: + fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+ + "reach nothing. On the machine: systemctl start %s\n", "", carried.Interface, + "wg-quick@"+carried.Interface) + default: + fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface) + } + if said && carried.Note != "" { + fmt.Printf(" %-17s %s\n", "", carried.Note) + } + if said && carried.Kept != "" { + fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept) + } + return nil +} + func orNone(s string) string { if s == "" { return "none reported" @@ -213,6 +255,28 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+ "never by the flip:\n%s", node, strings.Join(holding, "\n")) } + // And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR + // 0105): the flip loads the derived filter and retires the found firewall, and a machine the + // mesh has no record of is not one the filter admits — it would go dark. + if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil { + return "", err + } else if adopted && hubName == node { + carried, err := inv.CarriedPeers(ctx) + if err != nil { + return "", err + } + var waiting []string + for _, c := range carried { + if c.EnrolledAs == "" { + waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address)) + } + } + if len(waiting) > 0 { + return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+ + "converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s", + node, strings.Join(waiting, "\n")) + } + } shelf, err := inv.Catalogue(ctx) if err != nil { diff --git a/cmd/mesh-controller/network.go b/cmd/mesh-controller/network.go index 7f62c24..50ef8b3 100644 --- a/cmd/mesh-controller/network.go +++ b/cmd/mesh-controller/network.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "sort" + "strconv" "strings" "time" @@ -21,11 +22,28 @@ import ( // cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: // nothing in it was wrong, and no one edit was the one that should have been a new file. -func overlayCIDR() string { - if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" { - return v +// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another. +const DefaultOverlayCIDR = "10.42.0.0/16" + +// overlayRange is the range the mesh allocates node addresses from. +// +// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it +// found is at that tunnel's address, its peers are at theirs, and every node's address is +// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived +// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel, +// the range genesis was told, or the default. +func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) { + tunnel, _, adopted, err := inv.AdoptedTunnel(ctx) + if err != nil { + return "", err } - return "10.42.0.0/16" + if adopted { + return tunnel.Range, nil + } + if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" { + return v, nil + } + return DefaultOverlayCIDR, nil } func overlayCommand(ctx context.Context, args []string) error { @@ -110,16 +128,46 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) } } + // A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and + // the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would + // have the mesh's interface up where no peer is listening for it. + found, err := inv.NodeByName(ctx, node) + if err != nil { + return err + } + var tunnel inventory.Tunnel + adoptsTunnel := false + if *hub && found.Adopted { + if t, err := inv.TunnelOf(ctx, node); err == nil { + tunnel = t + placed, _ := inv.Overlays(ctx) + for _, o := range placed { + if o.Name == node && o.Key == t.PublicKey { + adoptsTunnel = true + } + } + } else if !errors.Is(err, inventory.ErrNoTunnel) { + return err + } + } + if adoptsTunnel { + if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) { + return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+ + "its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+ + "endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint) + } + } + // Declared, all three. The address is evidence of reachability and is not the fact, and hub // election by address prefix fails silently (novox/hq ADR 0007). if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil { return err } - found, err := inv.NodeByName(ctx, node) + cidr, err := overlayRange(ctx, inv) if err != nil { return err } - address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR()) + address, err := inv.AssignAddress(ctx, found.ID, cidr) if err != nil { return err } @@ -128,6 +176,10 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) fmt.Println(" credentials issued for it before this placement keep their old broker address —" + " `module issue` them again and push (novox/hq issue 059)") switch { + case adoptsTunnel: + fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+ + "%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port, + len(tunnel.Peers)) case *hub: fmt.Println(" the hub — every node not sharing a site routes through it") case *endpoint == "": @@ -154,15 +206,47 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, if err != nil { return nil, err } + // The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105). + tunnels, err := inv.Tunnels(ctx) + if err != nil { + return nil, err + } + carried, err := inv.CarriedPeers(ctx) + if err != nil { + return nil, err + } nodes := make([]overlay.Node, 0, len(places)) for _, p := range places { if !on[p.Name] { continue } - nodes = append(nodes, overlay.Node{ + n := overlay.Node{ Name: p.Name, Key: p.Key, Endpoint: p.Endpoint, Site: p.Site, Hub: p.Hub, Address: p.Address, - }) + } + if t, takes := tunnels[p.Name]; takes && t.NodeAdopted { + // Only an adopted node is told to take the found unit over: on a converged one there + // is nothing found to keep, and the host refuses the field. The range and the carried + // peers do not depend on the mode; the takeover does. + // + // **Refused, not composed, when the hub's record disagrees with the tunnel.** A + // declaration that stopped the found unit and raised the mesh's interface on another + // port or address would leave every peer dark while reporting the tunnel taken — so a + // hub placed before it took the tunnel over (or at the wrong port) is named here, and + // nothing is sent until it is re-placed. + if wrong := disagrees(p, t.Tunnel); wrong != "" { + return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+ + "Re-place it — `overlay place %s --hub --endpoint :%d …` — and push again; "+ + "nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port) + } + n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config} + } + if p.Hub { + for _, c := range carried { + n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address}) + } + } + nodes = append(nodes, n) } if len(nodes) == 0 { // Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this @@ -170,7 +254,11 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool, // "no hub" to somebody who never asked for a network would be a lie about the cause. return overlay.Empty(), nil } - g, err := overlay.From(nodes, overlayCIDR(), "") + cidr, err := overlayRange(ctx, inv) + if err != nil { + return nil, err + } + g, err := overlay.From(nodes, cidr, "") if g != nil { // The artifact store, as this network reaches it. Found rather than configured: the // provider is whichever module offers it, on whichever machine holds that module — and if @@ -292,6 +380,17 @@ func overlayShow(ctx context.Context, open *stores) error { return nil } + // The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105): + // listed apart from the nodes, because they are peers of the tunnel and not nodes of the + // mesh until they enrol — and once one has, it is listed as the node it became. + tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx) + if err != nil { + return err + } + carried, err := open.inventory.CarriedPeers(ctx) + if err != nil { + return err + } for _, n := range nodes { place := n.Address if place == "" { @@ -301,22 +400,74 @@ func overlayShow(ctx context.Context, open *stores) error { } fmt.Printf("%-16s %-14s", n.Name, place) switch { + case n.Hub && adopted: + fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)", + tunnel.Interface, tunnel.Range, tunnel.Port) + case n.Hub && hubName == n.Name && tunnel.Interface != "": + fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+ + "`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface) case n.Hub: - fmt.Print(" hub") + fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " + + "`mesh-host overlay take --tunnel ` there adopts it (novox/hq ADR 0105)") case !n.Reachable(): fmt.Print(" not dialable") } if n.Site != "" { fmt.Printf(" at %s", n.Site) } + if n.TakesOver != nil && !n.Hub { + fmt.Printf(" takes over %s", n.TakesOver.Interface) + } fmt.Println() for _, p := range computed[n.Name] { fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why) } } + if len(carried) > 0 { + fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName) + for _, c := range carried { + state := "not yet enrolled" + if c.EnrolledAs != "" { + state = "enrolled as " + c.EnrolledAs + ", which keeps this address" + } + fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state) + } + } return nil } +// disagrees says how a node's placement differs from the tunnel it takes over — its address not +// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree. +func disagrees(p inventory.Overlay, t inventory.Tunnel) string { + var wrong []string + want := t.Address + if i := strings.Index(want, "/"); i >= 0 { + want = want[:i] + } + if p.Address != want { + wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want)) + } + if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) { + wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port)) + } + return strings.Join(wrong, "; ") +} + +func orNothing(s string) string { + if s == "" { + return "unset" + } + return s +} + +// portOfEndpoint is the port in host:port, or empty. +func portOfEndpoint(endpoint string) string { + if i := strings.LastIndex(endpoint, ":"); i >= 0 { + return endpoint[i+1:] + } + return "" +} + // SilentFor is how long a node may be quiet before the mesh says so. // // A node speaks every minute, so three of them missed is a gap rather than a slow one. The number diff --git a/cmd/mesh-controller/network_test.go b/cmd/mesh-controller/network_test.go index ec42473..5370e2e 100644 --- a/cmd/mesh-controller/network_test.go +++ b/cmd/mesh-controller/network_test.go @@ -108,3 +108,130 @@ func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) { t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names) } } + +// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from +// the tunnel the hub holds — never stored anywhere else. +func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + t.Setenv(OverlayCIDRVar, "10.99.0.0/16") + + before, err := overlayRange(ctx, inv) + if err != nil || before != "10.99.0.0/16" { + t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err) + } + + // The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found + // tunnel's key, and presenting the tunnel. + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + hub, err := inv.NodeByName(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + const key = "THE-TUNNELS-KEY=========================" + if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil { + t.Fatal(err) + } + if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{ + Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900, + Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key, + Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}}, + }); err != nil { + t.Fatal(err) + } + + after, err := overlayRange(ctx, inv) + if err != nil || after != "192.0.2.0/24" { + t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err) + } + + // A placement whose endpoint is on another port than the tunnel's is refused: the peers dial + // the tunnel's port. + err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"}) + if err == nil || !strings.Contains(err.Error(), "51900") { + t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err) + } + // On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before. + if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil { + t.Fatal(err) + } + if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil { + t.Fatal(err) + } + if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" { + t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address) + } + + // And the hub's declaration carries the peer and the takeover. + nodes, computed, err := graph(ctx, open) + if err != nil { + t.Fatal(err) + } + var hubNode overlay.Node + for _, n := range nodes { + if n.Name == "anchor" { + hubNode = n + } + } + if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" { + t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode) + } + carried := false + for _, p := range computed["anchor"] { + if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" { + carried = true + } + } + if !carried { + t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"]) + } +} + +// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an +// endpoint port that differs would have the host stop the found interface and raise the mesh's +// where no peer is listening. +func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + hub, err := inv.NodeByName(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + const key = "THE-TUNNELS-KEY=========================" + if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil { + t.Fatal(err) + } + if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{ + Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900, + Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil { + t.Fatal(err) + } + // aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the + // tunnel over. + _, _, err = graph(ctx, open) + if err == nil { + t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's") + } + for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + // Re-placed on the tunnel, it composes. + if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil { + t.Fatal(err) + } + if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil { + t.Fatal(err) + } + if _, _, err := graph(ctx, open); err != nil { + t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err) + } +} diff --git a/internal/identity/fortest.go b/internal/identity/fortest.go new file mode 100644 index 0000000..9542c27 --- /dev/null +++ b/internal/identity/fortest.go @@ -0,0 +1,68 @@ +package identity + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/store" +) + +// ForTest is a fresh, migrated identity store in a database of its own, dropped when the test +// ends. Exported for the same reason inventory.ForTest is: the check that a node's signed word +// is verified against the key the mesh recorded lives beside the link, and a second copy of this +// would be a second thing to keep true. It takes a *testing.T, so nothing that is not a test can +// call it. +func ForTest(t *testing.T) *Identity { + t.Helper() + admin := os.Getenv("MESH_TEST_POSTGRES") + if admin == "" { + t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one") + } + name := fmt.Sprintf("ident_%d_%s", time.Now().UnixNano()%1_000_000, + strings.ToLower(strings.NewReplacer("/", "", "-", "").Replace(t.Name()))) + if len(name) > 60 { + name = name[:60] + } + conn, err := pgx.Connect(t.Context(), admin) + if err != nil { + t.Fatalf("cannot reach the test PostgreSQL: %v", err) + } + if _, err := conn.Exec(t.Context(), "create database "+name); err != nil { + t.Fatalf("cannot create %s: %v", name, err) + } + conn.Close(t.Context()) + + cut := strings.LastIndex(admin, "/") + t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable") + + ident, err := Open(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ident.Close() + c, err := pgx.Connect(context.Background(), admin) + if err != nil { + return + } + defer c.Close(context.Background()) + _, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)") + }) + if err := ident.Ready(t.Context(), 20*time.Second); err != nil { + t.Fatal(err) + } + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + if _, err := ident.store.Migrate(t.Context(), migrations); err != nil { + t.Fatal(err) + } + return ident +} diff --git a/internal/inventory/addresses.go b/internal/inventory/addresses.go index 3826e0b..44214df 100644 --- a/internal/inventory/addresses.go +++ b/internal/inventory/addresses.go @@ -16,25 +16,64 @@ import ( // node's peer list to chase it, and an address that moves is the thing declaring the hub was // meant to stop. // -// Allocated in order from the range, taking the lowest free one. Not random: a person reading a -// peer list should be able to guess which node an address belongs to, and reuse of a released -// address is a smaller problem than a list nobody can hold in their head. +// **The adopted tunnel's addresses come first** (novox/hq ADR 0105). The hub that took over a +// tunnel is at the tunnel's own address. A node enrolling with a key the tunnel already routed +// to keeps the address the tunnel had for it — nothing a peer knows changes. And an address the +// tunnel holds for a peer that has not enrolled is never handed to anyone else: that peer is +// still reaching the hub at it. +// +// The rest is allocated in order from the range, taking the lowest free one. Not random: a person +// reading a peer list should be able to guess which node an address belongs to, and reuse of a +// released address is a smaller problem than a list nobody can hold in their head. func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) { prefix, err := netip.ParsePrefix(cidr) if err != nil { return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err) } - var existing *string + var existing, key *string + var name string + var hub bool if err := i.store.Pool().QueryRow(ctx, - `select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil { + `select name, host(overlay_address), overlay_key, is_hub from node where id = $1`, node). + Scan(&name, &existing, &key, &hub); err != nil { return "", err } if existing != nil && *existing != "" { return *existing, nil } + tunnel, hubName, adopted, err := i.AdoptedTunnel(ctx) + if err != nil { + return "", err + } + if adopted && hub && hubName == name { + address, err := netip.ParsePrefix(tunnel.Address) + if err != nil { + return "", fmt.Errorf("the adopted tunnel's address %q: %w", tunnel.Address, err) + } + return i.place(ctx, node, address.Addr().String()) + } + carried, err := i.CarriedPeers(ctx) + if err != nil { + return "", err + } taken := map[string]bool{} + if adopted { + // The tunnel's own address is the hub's whether or not the hub has been placed yet. + if address, err := netip.ParsePrefix(tunnel.Address); err == nil { + taken[address.Addr().String()] = true + } + } + for _, p := range carried { + if key != nil && p.PublicKey == *key { + // The tunnel already routes to this key: the node keeps that address, and the peer + // notices nothing when its machine enrols. + return i.place(ctx, node, p.Address) + } + taken[p.Address] = true + } + rows, err := i.store.Pool().Query(ctx, `select host(overlay_address) from node where overlay_address is not null`) if err != nil { @@ -58,12 +97,7 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin candidate := prefix.Masked().Addr().Next() for prefix.Contains(candidate) { if !taken[candidate.String()] { - if _, err := i.store.Pool().Exec(ctx, - `update node set overlay_address = $2::inet where id = $1`, - node, candidate.String()); err != nil { - return "", err - } - return candidate.String(), nil + return i.place(ctx, node, candidate.String()) } candidate = candidate.Next() } @@ -72,5 +106,13 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin // halfway through assigning one node. return "", fmt.Errorf( "every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+ - "range and renumbering it is a deliberate act", cidr, node) + "range and renumbering it is a deliberate act", cidr, name) +} + +func (i *Inventory) place(ctx context.Context, node, address string) (string, error) { + if _, err := i.store.Pool().Exec(ctx, + `update node set overlay_address = $2::inet where id = $1`, node, address); err != nil { + return "", err + } + return address, nil } diff --git a/internal/inventory/migrations/0031-the-tunnel-a-node-found.sql b/internal/inventory/migrations/0031-the-tunnel-a-node-found.sql new file mode 100644 index 0000000..88cb466 --- /dev/null +++ b/internal/inventory/migrations/0031-the-tunnel-a-node-found.sql @@ -0,0 +1,27 @@ +-- The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105). +-- +-- On an adopted node that is the hub, the private network takes over the tunnel it finds: its +-- key, its port, its address and range, and every peer. The node presents what it found when it +-- enrols -- the same moment it presents its keys, because the found tunnel's key IS its key on the +-- private network from then on -- and the mesh composes every address from it. + +-- What the node presented: interface, unit and configuration path, port, address and range, and +-- the tunnel's public key. The private key never travels; the node keeps it as its own overlay +-- key. Null on a node that found no tunnel, which is every converged one. +alter table node add column tunnel jsonb; + +-- The node's last account of carrying it: the found interface down and disabled, the mesh's up +-- in its place. Null until the node says so. +alter table node add column tunnel_carried jsonb; + +-- The peers the found tunnel had: a public key and the address the tunnel routed to it. Peers of +-- the tunnel, not nodes of the mesh, until they enrol -- a machine the mesh has no record of, whose +-- identity precedes its enrolment. One row per key, and one address per key on one tunnel. +create table tunnel_peer ( + node uuid not null references node(id) on delete cascade, + public_key text not null, + address inet not null, + since timestamptz not null default now(), + primary key (node, public_key), + unique (node, address) +); diff --git a/internal/inventory/tunnel.go b/internal/inventory/tunnel.go new file mode 100644 index 0000000..39ceeb7 --- /dev/null +++ b/internal/inventory/tunnel.go @@ -0,0 +1,375 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/netip" + "strings" + "time" + + "github.com/jackc/pgx/v5" +) + +// The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105). +// +// On an adopted node that is the hub, the private network takes over the tunnel it finds: its +// private key, its port, its address and range, and every peer the found interface had. The node +// presents what it found when it enrols, in the same breath as its keys — the found tunnel's key is +// its key on the private network from then on — and the mesh composes every address from it: the +// hub's is the tunnel's, the range is the tunnel's, and a peer that enrols keeps the address the +// tunnel already had for its key. + +// Tunnel is what a node found on its machine, as it presented it. No private key: the node keeps +// it as its own overlay key, sealed like any own secret, and the mesh records only the public half +// — which is then the node's overlay key too. +type Tunnel struct { + // Interface, Unit and Config are what the host takes over: the found interface, the unit + // that raised it, and its configuration file, which is kept like any held file. + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + // Port is the port the found interface listened on — one the hosting provider already lets + // through, which is why it is worth taking. + Port int `json:"port"` + // Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the + // network that prefix names, 192.0.2.0/24. + Address string `json:"address"` + Range string `json:"range"` + // PublicKey is the found interface's, which every peer knows the tunnel by. + PublicKey string `json:"public_key"` + // Peers are the found interface's peers: each a public key and the address the tunnel routed + // to it. + Peers []TunnelPeer `json:"peers,omitempty"` + // At is when the node presented it; zero on a tunnel not yet recorded. + At time.Time `json:"at,omitempty"` +} + +// TunnelPeer is one peer of a found tunnel. +type TunnelPeer struct { + PublicKey string `json:"public_key"` + // Address is the one host address the tunnel routed to the peer, without a prefix. + Address string `json:"address"` +} + +// Carried is what a node last said about carrying the tunnel it found: the found interface down +// and disabled, the mesh's up in its place with the found key. +type Carried struct { + Interface string `json:"interface"` + Port int `json:"port"` + Range string `json:"range"` + Peers int `json:"peers"` + // State is one of the CarriedStates: the found interface is still up and the mesh's is not + // (not taken), the found one is down and the mesh's up with its key (taken), or the found one + // is down and the mesh's is not up — the one state where the peers reach nothing. Note is + // what the host did about it, when it did something. Kept is where the found configuration's + // original was kept. + State string `json:"state"` + Note string `json:"note,omitempty"` + Kept string `json:"kept,omitempty"` + At time.Time `json:"at"` +} + +// The states a carried tunnel's account can be in, as the host says them. +const ( + CarriedNotTaken = "not-taken" + CarriedTaken = "taken" + CarriedDown = "down" +) + +// CarriedPeer is one peer of the adopted tunnel as the mesh holds it: a peer of the tunnel, and +// — once a node enrols with that key — a node of the mesh as well. +type CarriedPeer struct { + PublicKey string + Address string + // EnrolledAs names the node that enrolled with this key, or is empty while none has. + EnrolledAs string +} + +// ErrNoTunnel is asking about a tunnel on a node that presented none. +var ErrNoTunnel = errors.New("that node presented no tunnel") + +// RecordTunnel keeps what a node presented, replacing what was there: the question is the tunnel +// as the node found it now. The peers are replaced whole for the same reason. +func (i *Inventory) RecordTunnel(ctx context.Context, nodeID string, t Tunnel) error { + if strings.TrimSpace(t.Interface) == "" || strings.TrimSpace(t.PublicKey) == "" { + return errors.New("a found tunnel names its interface and its public key, and this names neither") + } + if _, err := netip.ParsePrefix(t.Range); err != nil { + return fmt.Errorf("the found tunnel's range %q is not a range: %w", t.Range, err) + } + address, err := netip.ParsePrefix(t.Address) + if err != nil { + return fmt.Errorf("the found tunnel's address %q is not an address with a prefix: %w", t.Address, err) + } + peers := make([]TunnelPeer, 0, len(t.Peers)) + for _, p := range t.Peers { + host, single := peerHost(p.Address) + if !single { + // A peer routed a range rather than one address is a spoke's view of its hub — the + // predecessor gives a spoke the whole subnet through the hub — and a hub is not a peer + // the mesh carries. Skipped, not refused: a spoke enrols with what it found, and only + // the hub's peers are ever carried (novox/hq ADR 0105). + continue + } + if !address.Masked().Contains(host) { + return fmt.Errorf("the found tunnel's peer %s is routed at %s, outside the tunnel's %s", + shortKey(p.PublicKey), host, t.Range) + } + peers = append(peers, TunnelPeer{PublicKey: p.PublicKey, Address: host.String()}) + } + t.Peers = nil + t.At = time.Now().UTC() + raw, err := json.Marshal(t) + if err != nil { + return err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, `update node set tunnel = $2 where id = $1`, nodeID, raw); err != nil { + return err + } + if _, err := tx.Exec(ctx, `delete from tunnel_peer where node = $1`, nodeID); err != nil { + return err + } + for _, p := range peers { + if _, err := tx.Exec(ctx, + `insert into tunnel_peer (node, public_key, address) values ($1, $2, $3::inet)`, + nodeID, p.PublicKey, p.Address); err != nil { + return fmt.Errorf("recording the found tunnel's peer %s: %w", shortKey(p.PublicKey), err) + } + } + return tx.Commit(ctx) +} + +// peerHost is the one host address a peer's allowed address names — a bare address, or a /32 +// (or /128) — and false for anything wider or unreadable: a peer routed a whole range is not a +// machine with an address the mesh could give a node. +func peerHost(allowed string) (netip.Addr, bool) { + allowed = strings.TrimSpace(allowed) + if a, err := netip.ParseAddr(allowed); err == nil { + return a, true + } + p, err := netip.ParsePrefix(allowed) + if err != nil || !p.IsSingleIP() { + return netip.Addr{}, false + } + return p.Addr(), true +} + +func shortKey(key string) string { + if len(key) > 8 { + return key[:8] + "…" + } + return key +} + +// TunnelOf is the tunnel a node presented, with its peers, or ErrNoTunnel. +func (i *Inventory) TunnelOf(ctx context.Context, name string) (Tunnel, error) { + var raw []byte + var id string + err := i.store.Pool().QueryRow(ctx, + `select id, tunnel from node where name = $1`, name).Scan(&id, &raw) + if errors.Is(err, pgx.ErrNoRows) { + return Tunnel{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + if err != nil { + return Tunnel{}, err + } + if len(raw) == 0 { + return Tunnel{}, fmt.Errorf("%w: %s", ErrNoTunnel, name) + } + var t Tunnel + if err := json.Unmarshal(raw, &t); err != nil { + return Tunnel{}, err + } + t.Peers, err = i.tunnelPeers(ctx, id) + return t, err +} + +func (i *Inventory) tunnelPeers(ctx context.Context, nodeID string) ([]TunnelPeer, error) { + rows, err := i.store.Pool().Query(ctx, + `select public_key, host(address) from tunnel_peer where node = $1 order by address`, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []TunnelPeer + for rows.Next() { + var p TunnelPeer + if err := rows.Scan(&p.PublicKey, &p.Address); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// AdoptedTunnel is the tunnel the mesh's private network runs over, if the hub adopted one: the +// hub's found tunnel, when the hub's overlay key is the tunnel's. Absent, the mesh runs on its own +// range — and a hub that found a tunnel but holds another key did not adopt it, which `overlay +// show` says. +// +// The condition on the key is the condition of the whole record: a hub raised with a key of its +// own would drop every peer's packets on the found port (novox/hq ADR 0105, option 2), so the +// tunnel is adopted only when the hub answers to the key its peers know. **Not a condition on the +// node's mode**: the range and the carried peers are facts of the mesh once the tunnel is taken, +// and converging the hub — which flips its mode — must not renumber the mesh or drop the peers +// still reaching it. +func (i *Inventory) AdoptedTunnel(ctx context.Context) (Tunnel, string, bool, error) { + var name string + var key *string + err := i.store.Pool().QueryRow(ctx, + `select name, overlay_key from node where is_hub and tunnel is not null`). + Scan(&name, &key) + if errors.Is(err, pgx.ErrNoRows) { + return Tunnel{}, "", false, nil + } + if err != nil { + return Tunnel{}, "", false, err + } + t, err := i.TunnelOf(ctx, name) + if err != nil { + return Tunnel{}, "", false, err + } + if key == nil || *key != t.PublicKey { + return t, name, false, nil + } + return t, name, true, nil +} + +// CarriedPeers is every peer of the adopted tunnel, with the node that enrolled under its key +// where one has: peers of the tunnel, and nodes of the mesh once they enrol. Empty when the hub +// adopted no tunnel. +func (i *Inventory) CarriedPeers(ctx context.Context) ([]CarriedPeer, error) { + rows, err := i.store.Pool().Query(ctx, + `select p.public_key, host(p.address), coalesce(n.name, '') + from tunnel_peer p + join node hub on hub.id = p.node and hub.is_hub + and hub.tunnel is not null and hub.overlay_key = hub.tunnel->>'public_key' + left join node n on n.overlay_key = p.public_key + order by p.address`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []CarriedPeer + for rows.Next() { + var p CarriedPeer + if err := rows.Scan(&p.PublicKey, &p.Address, &p.EnrolledAs); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// FoundTunnel is a node's found tunnel with the node's mode, for composing: the takeover is +// declared to an adopted node only, since only there is a found unit kept to be stopped. +type FoundTunnel struct { + Tunnel + NodeAdopted bool +} + +// Tunnels is every node's found tunnel by node name, for the ones whose overlay key is the +// tunnel's — the ones whose private network takes it over. A found tunnel under another key is +// left running beside the mesh's, and ADR 0100's rule that the ranges differ applies to it. +func (i *Inventory) Tunnels(ctx context.Context) (map[string]FoundTunnel, error) { + rows, err := i.store.Pool().Query(ctx, + `select name, tunnel, adopted from node + where tunnel is not null and overlay_key = tunnel->>'public_key'`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]FoundTunnel{} + for rows.Next() { + var name string + var raw []byte + var adopted bool + if err := rows.Scan(&name, &raw, &adopted); err != nil { + return nil, err + } + var t Tunnel + if err := json.Unmarshal(raw, &t); err != nil { + return nil, err + } + out[name] = FoundTunnel{Tunnel: t, NodeAdopted: adopted} + } + return out, rows.Err() +} + +// ErrStaleRekey is a rekey that names a previous overlay key other than the one recorded: a +// replay of a rekey already done, or one made against a record that has since moved on. +var ErrStaleRekey = errors.New("the rekey names a previous overlay key that is not the node's current one") + +// Rekey records that a node took a found tunnel's key as its overlay key after enrolling (novox/hq +// ADR 0105): the key and the tunnel are recorded as enrolment would have, and a hub is moved to the +// tunnel's address so nothing derived from it is stale. The caller has verified the node signed +// for this; what is checked here is that it follows the record — `previous` is the overlay key the +// node holds now — so the same message cannot be applied twice. +func (i *Inventory) Rekey(ctx context.Context, nodeID, previous, key string, t Tunnel) error { + if key != t.PublicKey { + return errors.New("a rekey takes a tunnel over with the tunnel's own key, and this names another") + } + var current *string + var hub bool + if err := i.store.Pool().QueryRow(ctx, + `select overlay_key, is_hub from node where id = $1`, nodeID).Scan(¤t, &hub); err != nil { + return err + } + if (current == nil && previous != "") || (current != nil && *current != previous) { + return ErrStaleRekey + } + if err := i.RecordOverlayKey(ctx, nodeID, key); err != nil { + return err + } + if err := i.RecordTunnel(ctx, nodeID, t); err != nil { + return err + } + if hub { + address, err := netip.ParsePrefix(t.Address) + if err != nil { + return err + } + if _, err := i.place(ctx, nodeID, address.Addr().String()); err != nil { + return err + } + } + return nil +} + +// RecordCarriedTunnel keeps what a node last said about carrying its found tunnel. +func (i *Inventory) RecordCarriedTunnel(ctx context.Context, nodeID string, c Carried) error { + c.At = time.Now().UTC() + raw, err := json.Marshal(c) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, `update node set tunnel_carried = $2 where id = $1`, nodeID, raw) + return err +} + +// CarriedTunnelOf is a node's last account of carrying its found tunnel, and whether it ever gave one. +func (i *Inventory) CarriedTunnelOf(ctx context.Context, name string) (Carried, bool, error) { + var raw []byte + err := i.store.Pool().QueryRow(ctx, `select tunnel_carried from node where name = $1`, name).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return Carried{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + if err != nil { + return Carried{}, false, err + } + if len(raw) == 0 { + return Carried{}, false, nil + } + var c Carried + if err := json.Unmarshal(raw, &c); err != nil { + return Carried{}, false, err + } + return c, true, nil +} diff --git a/internal/inventory/tunnel_test.go b/internal/inventory/tunnel_test.go new file mode 100644 index 0000000..9498ad4 --- /dev/null +++ b/internal/inventory/tunnel_test.go @@ -0,0 +1,279 @@ +package inventory + +import ( + "errors" + "strings" + "testing" +) + +// novox/hq ADR 0105: the mesh adopts the predecessor's tunnel in place. The controller reads the +// hub's address and range from the adopted tunnel, assigns an enrolling node the address its key +// already had, and refuses to hand out an address the tunnel already holds. + +const ( + tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key=" + peerTwo = "PEER-KEY-two=============================" + peerThree = "PEER-KEY-three===========================" +) + +// theFoundTunnel is what a hub presents at enrolment: the predecessor's interface on a +// documentation range, with two peers each routed one address. +func theFoundTunnel() Tunnel { + return Tunnel{ + Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", + Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey, + Peers: []TunnelPeer{{PublicKey: peerTwo, Address: "192.0.2.2/32"}, + {PublicKey: peerThree, Address: "192.0.2.3"}}, + } +} + +// anAdoptedHub is an adopted node that enrolled with the found tunnel's key and presented the +// tunnel, then was placed as the hub — the order genesis does it in. +func anAdoptedHub(t *testing.T, inv *Inventory) Node { + t.Helper() + hub, err := inv.AddNodeAs(t.Context(), "anchor", true) + if err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(t.Context(), hub.ID, tunnelKey); err != nil { + t.Fatal(err) + } + if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil { + t.Fatal(err) + } + return hub +} + +func TestTheHubsAddressAndRangeComeFromTheAdoptedTunnel(t *testing.T) { + inv := fresh(t) + hub := anAdoptedHub(t, inv) + + tunnel, name, adopted, err := inv.AdoptedTunnel(t.Context()) + if err != nil { + t.Fatal(err) + } + if !adopted || name != "anchor" || tunnel.Range != "192.0.2.0/24" || tunnel.Port != 51900 { + t.Fatalf("the adopted tunnel did not read back: adopted=%t on %s, %+v", adopted, name, tunnel) + } + if len(tunnel.Peers) != 2 || tunnel.Peers[0].Address != "192.0.2.2" || tunnel.Peers[1].Address != "192.0.2.3" { + t.Fatalf("the peers did not read back as one host address each: %+v", tunnel.Peers) + } + + // Whatever range the caller would allocate from, the hub is at the tunnel's own address. + address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16") + if err != nil { + t.Fatal(err) + } + if address != "192.0.2.1" { + t.Fatalf("the hub was given %s, not the address the tunnel it took over had", address) + } +} + +func TestAnEnrollingNodeKeepsTheAddressTheTunnelHadForItsKey(t *testing.T) { + inv := fresh(t) + anAdoptedHub(t, inv) + + // A predecessor machine enrols: its host took its found interface's key as its overlay key, + // which is the key the hub's tunnel already routes to. + peer, err := inv.AddNodeAs(t.Context(), "home-server", true) + if err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(t.Context(), peer.ID, peerThree); err != nil { + t.Fatal(err) + } + address, err := inv.AssignAddress(t.Context(), peer.ID, "192.0.2.0/24") + if err != nil { + t.Fatal(err) + } + if address != "192.0.2.3" { + t.Fatalf("the enrolling peer was given %s, not the 192.0.2.3 the tunnel had for its key", address) + } + + carried, err := inv.CarriedPeers(t.Context()) + if err != nil { + t.Fatal(err) + } + byKey := map[string]CarriedPeer{} + for _, c := range carried { + byKey[c.PublicKey] = c + } + if byKey[peerThree].EnrolledAs != "home-server" || byKey[peerTwo].EnrolledAs != "" { + t.Fatalf("the registry cannot say which peer is a node now: %+v", carried) + } +} + +func TestAFreshNodeIsNeverGivenAnAddressTheTunnelHolds(t *testing.T) { + inv := fresh(t) + anAdoptedHub(t, inv) + + // .1 is the hub, .2 and .3 are peers of the tunnel that have not enrolled: a new machine with + // a key of its own gets the next one, from the same range. + fresh, err := inv.AddNode(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(t.Context(), fresh.ID, "A-KEY-OF-ITS-OWN========================"); err != nil { + t.Fatal(err) + } + address, err := inv.AssignAddress(t.Context(), fresh.ID, "192.0.2.0/24") + if err != nil { + t.Fatal(err) + } + if address != "192.0.2.4" { + t.Fatalf("a fresh node was given %s; 192.0.2.2 and .3 are the tunnel's peers and .1 its hub", address) + } +} + +func TestATunnelUnderAnotherKeyIsNotAdopted(t *testing.T) { + // A hub whose overlay key is not the found tunnel's would drop every peer's packets on the + // found port (ADR 0105, option 2). Such a tunnel is recorded and not adopted: the mesh keeps + // its own range, and ADR 0100's non-overlap rule stands for it. + inv := fresh(t) + hub, err := inv.AddNodeAs(t.Context(), "anchor", true) + if err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(t.Context(), hub.ID, "THE-MESHS-OWN-KEY======================="); err != nil { + t.Fatal(err) + } + if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51820", "hosting", true, ""); err != nil { + t.Fatal(err) + } + if _, _, adopted, err := inv.AdoptedTunnel(t.Context()); err != nil || adopted { + t.Fatalf("a tunnel under another key was adopted (err %v)", err) + } + if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 0 { + t.Fatalf("peers of a tunnel that was not adopted are carried: %+v (err %v)", carried, err) + } + if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" { + t.Fatalf("the hub was given %s (err %v); it should allocate from the mesh's own range", address, err) + } +} + +func TestAPeerRoutedARangeIsNotCarried(t *testing.T) { + // A peer routed a whole range is a spoke's view of its hub, never a machine with an address + // the mesh could carry: skipped, and the single-address peers beside it kept. + inv := fresh(t) + hub, err := inv.AddNodeAs(t.Context(), "anchor", true) + if err != nil { + t.Fatal(err) + } + found := theFoundTunnel() + found.Peers = append(found.Peers, TunnelPeer{PublicKey: "WIDE", Address: "192.0.2.0/24"}) + if err := inv.RecordTunnel(t.Context(), hub.ID, found); err != nil { + t.Fatal(err) + } + got, err := inv.TunnelOf(t.Context(), "anchor") + if err != nil || len(got.Peers) != 2 { + t.Fatalf("the range-routed peer was carried, or the others dropped: %+v %v", got.Peers, err) + } + // A single address outside the tunnel's range is still refused: it is not a peer this tunnel + // routes to. + found.Peers = []TunnelPeer{{PublicKey: "ELSEWHERE", Address: "198.51.100.7/32"}} + if err := inv.RecordTunnel(t.Context(), hub.ID, found); err == nil || !strings.Contains(err.Error(), "outside") { + t.Fatalf("a peer outside the range was recorded: %v", err) + } +} + +// A predecessor spoke's tunnel has one peer — the hub — routed the whole range. Its enrolment must +// not fail on it: only the hub's peers are ever carried, so a range-routed peer is skipped. +func TestASpokesTunnelEnrolsWithItsHubPeerSkipped(t *testing.T) { + inv := fresh(t) + anAdoptedHub(t, inv) + spoke, err := inv.AddNodeAs(t.Context(), "home-server", true) + if err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(t.Context(), spoke.ID, peerThree); err != nil { + t.Fatal(err) + } + if err := inv.RecordTunnel(t.Context(), spoke.ID, Tunnel{ + Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900, + Address: "192.0.2.3/24", Range: "192.0.2.0/24", PublicKey: peerThree, + Peers: []TunnelPeer{{PublicKey: tunnelKey, Address: "192.0.2.0/24"}}, + }); err != nil { + t.Fatalf("a spoke-shaped tunnel was refused: %v", err) + } + got, err := inv.TunnelOf(t.Context(), "home-server") + if err != nil || len(got.Peers) != 0 { + t.Fatalf("the spoke's hub was recorded as a peer to carry: %+v %v", got.Peers, err) + } + // Nothing about the hub's carried peers changed: still two, one now enrolled. + carried, err := inv.CarriedPeers(t.Context()) + if err != nil || len(carried) != 2 { + t.Fatalf("carried peers: %+v %v", carried, err) + } + if address, err := inv.AssignAddress(t.Context(), spoke.ID, "192.0.2.0/24"); err != nil || address != "192.0.2.3" { + t.Fatalf("the spoke did not keep its address: %s %v", address, err) + } +} + +// Converging the hub flips its mode and nothing else: the range stays the tunnel's and the peers +// stay carried, or the mesh would renumber itself and drop the peers still reaching it. +func TestConvergingTheHubKeepsTheRangeAndTheCarriedPeers(t *testing.T) { + inv := fresh(t) + hub := anAdoptedHub(t, inv) + if _, err := inv.AssignAddress(t.Context(), hub.ID, "192.0.2.0/24"); err != nil { + t.Fatal(err) + } + if _, err := inv.Converge(t.Context(), "anchor"); err != nil { + t.Fatal(err) + } + tunnel, _, adopted, err := inv.AdoptedTunnel(t.Context()) + if err != nil || !adopted || tunnel.Range != "192.0.2.0/24" { + t.Fatalf("converging renumbered the mesh: adopted=%t %+v %v", adopted, tunnel, err) + } + if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 2 { + t.Fatalf("converging dropped the carried peers: %+v %v", carried, err) + } + found, err := inv.Tunnels(t.Context()) + if err != nil || found["anchor"].NodeAdopted { + t.Fatalf("a converged hub still reads as adopted for the takeover: %+v %v", found, err) + } +} + +// A hub that enrolled with a key of its own takes the tunnel over afterwards by rekeying: the key +// and the tunnel are recorded, the hub moves to the tunnel's address, and the same rekey applied +// again is stale. +func TestARekeyTakesTheTunnelOverAfterEnrolment(t *testing.T) { + inv := fresh(t) + hub, err := inv.AddNodeAs(t.Context(), "anchor", true) + if err != nil { + t.Fatal(err) + } + const own = "THE-MESHS-OWN-KEY=======================" + if err := inv.RecordOverlayKey(t.Context(), hub.ID, own); err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil { + t.Fatal(err) + } + if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" { + t.Fatalf("before the rekey the hub is on the mesh's own range: %s %v", address, err) + } + + if err := inv.Rekey(t.Context(), hub.ID, own, tunnelKey, theFoundTunnel()); err != nil { + t.Fatal(err) + } + _, _, adopted, err := inv.AdoptedTunnel(t.Context()) + if err != nil || !adopted { + t.Fatalf("the tunnel is not adopted after the rekey (%v)", err) + } + placed, err := inv.Overlays(t.Context()) + if err != nil || len(placed) != 1 || placed[0].Address != "192.0.2.1" || placed[0].Key != tunnelKey { + t.Fatalf("the hub did not move to the tunnel's address under the tunnel's key: %+v %v", placed, err) + } + if err := inv.Rekey(t.Context(), hub.ID, own, tunnelKey, theFoundTunnel()); !errors.Is(err, ErrStaleRekey) { + t.Fatalf("the same rekey applied again was not refused as stale: %v", err) + } + if err := inv.Rekey(t.Context(), hub.ID, tunnelKey, "ANOTHER-KEY=============================", theFoundTunnel()); err == nil { + t.Fatal("a rekey to a key that is not the tunnel's was accepted") + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index d4d71aa..54b8967 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -124,6 +124,29 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol "%s's overlay key could not be recorded: %w", node.Name, err) } } + // And the tunnel it found, whose key is the overlay key above (novox/hq ADR 0105). Recorded + // before the token is spent for the same reason as the keys: the first declaration this node + // receives is composed from it, and a hub enrolled without its tunnel would be placed at an + // address of the mesh's choosing rather than the tunnel's. + if request.Tunnel != nil { + if request.Tunnel.PublicKey != request.OverlayKey { + return EnrolReply{}, fmt.Errorf("%s presented a tunnel under key %s and an overlay key "+ + "that is not it; a tunnel is taken over with its own key or not at all", node.Name, + request.Tunnel.PublicKey) + } + peers := make([]inventory.TunnelPeer, 0, len(request.Tunnel.Peers)) + for _, p := range request.Tunnel.Peers { + peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address}) + } + if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{ + Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit, + Config: request.Tunnel.Config, Port: request.Tunnel.Port, + Address: request.Tunnel.Address, Range: request.Tunnel.Range, + PublicKey: request.Tunnel.PublicKey, Peers: peers, + }); err != nil { + return EnrolReply{}, fmt.Errorf("%s's found tunnel could not be recorded: %w", node.Name, err) + } + } // Spent once the node is complete in the store. if err := e.Inventory.Spend(ctx, secret, by); err != nil { @@ -158,6 +181,34 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol return reply, nil } +// rekey applies a verified rekey: the node's overlay key and tunnel are recorded as enrolment +// would have recorded them, and a hub moves to the tunnel's address. +func (e Enrolment) rekey(ctx context.Context, node inventory.Node, r Rekey) error { + if r.Tunnel == nil || r.OverlayKey == "" { + return fmt.Errorf("%s sent a rekey naming no tunnel or no key; refused", node.Name) + } + if e.Identity == nil { + return fmt.Errorf("%s sent a rekey and this mesh has no identity store to verify it against", node.Name) + } + if err := e.Identity.VerifyNode(ctx, node.ID, + RekeyProof(node.Name, r.Previous, r.OverlayKey, r.Tunnel), r.Proof); err != nil { + return fmt.Errorf("%s's rekey is not signed by %s's identity key; refused: %w", node.Name, node.Name, err) + } + peers := make([]inventory.TunnelPeer, 0, len(r.Tunnel.Peers)) + for _, p := range r.Tunnel.Peers { + peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address}) + } + err := e.Inventory.Rekey(ctx, node.ID, r.Previous, r.OverlayKey, inventory.Tunnel{ + Interface: r.Tunnel.Interface, Unit: r.Tunnel.Unit, Config: r.Tunnel.Config, Port: r.Tunnel.Port, + Address: r.Tunnel.Address, Range: r.Tunnel.Range, PublicKey: r.Tunnel.PublicKey, Peers: peers, + }) + if err != nil { + return fmt.Errorf("%s's rekey was not recorded: %w", node.Name, err) + } + log.Printf("%s took over the tunnel on %s: its overlay key is the tunnel's now", node.Name, r.Tunnel.Interface) + return nil +} + // claimant names the key presenting a token, so a claim can be held for it alone. func claimant(public ed25519.PublicKey) string { sum := sha256.Sum256(public) @@ -219,6 +270,26 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) { return err } } + // What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it. + if report.Tunnel != nil { + if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{ + Interface: report.Tunnel.Interface, Port: report.Tunnel.Port, Range: report.Tunnel.Range, + Peers: report.Tunnel.Peers, State: report.Tunnel.State, Note: report.Tunnel.Note, + Kept: report.Tunnel.Kept, + }); err != nil { + return err + } + } + // A node taking a found tunnel's key after enrolment (novox/hq ADR 0105). Verified against the + // node's live identity key before anything is written: the broker account authenticates the + // connection, the signature proves the node itself said it. Refused outright when the proof + // does not verify or is stale — a refusal, not "not now", so the node hears why. + if report.Rekey != nil { + if err := e.rekey(ctx, node, *report.Rekey); err != nil { + return err + } + return e.Inventory.Seen(ctx, node.ID) + } // A bare word that a node is there is not an account of what the machine did or holds: it // moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery), diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 0a7227d..d814d15 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -8,6 +8,8 @@ package link import ( "encoding/base64" + "strconv" + "strings" "time" ) @@ -71,12 +73,39 @@ type EnrolRequest struct { // node's public key could replay the spent token (novox/hq issue 083, on review). Proof []byte `json:"proof,omitempty"` + // Tunnel is the tunnel this node found on its machine and whose key it took as its overlay + // key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented + // with the keys because it is one of them — OverlayKey above is this tunnel's public key when + // it is set — and the mesh composes the hub's address, the range and every carried peer from + // it. Nil from a node that found none, which is every converged one. + Tunnel *Tunnel `json:"tunnel,omitempty"` + // Redelivered is set by the control plane, never sent: the broker handed this request over a // second time. Such a request does not finish an enrolment already spent — the first time may // have answered, and the node holds what it was told. Redelivered bool `json:"-"` } +// Tunnel is a found tunnel as a node presents it: everything but its private key, which the node +// keeps as its own overlay key and never sends. +type Tunnel struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + Port int `json:"port"` + Address string `json:"address"` + Range string `json:"range"` + PublicKey string `json:"public_key"` + Peers []TunnelPeer `json:"peers,omitempty"` +} + +// TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to +// it. +type TunnelPeer struct { + PublicKey string `json:"public_key"` + Address string `json:"address"` +} + // Signed is a declaration and the signature over it. // // The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies @@ -129,6 +158,60 @@ type Report struct { // Reachable is what can be reached on the machine now: every listening socket and every // published container port. Only an adopted node reports it; it is what converging previews. Reachable []Reach `json:"reachable,omitempty"` + + // Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR + // 0105): the interface, its port, range and peer count, whether the found interface is down + // and the mesh's up in its place, and where the found configuration's original was kept. + Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + + // Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq + // ADR 0105). A report carrying one is not an account of the machine: it moves the node's + // overlay key and tunnel and nothing else. + Rekey *Rekey `json:"rekey,omitempty"` +} + +// CarriedTunnel is a node's account of the tunnel it took over. State is "not-taken" (the found +// interface still up, the mesh's not), "taken" (the found one down, the mesh's up with its key) or +// "down" (the found one down and the mesh's not up: the peers reach nothing); Note is what the host +// did about it. +type CarriedTunnel struct { + Interface string `json:"interface"` + Port int `json:"port"` + Range string `json:"range"` + Peers int `json:"peers"` + State string `json:"state"` + Note string `json:"note,omitempty"` + Kept string `json:"kept,omitempty"` +} + +// Rekey is a node saying it took a found tunnel's key as its overlay key after enrolling (novox/hq +// ADR 0105) — the path for a hub that enrolled before the mesh knew to take a tunnel over, since +// re-enrolling would rotate every key the node holds. Carried in a report, on the node's own +// authenticated connection, and signed with its identity key over RekeyProof, so a report forged +// on a stolen broker account cannot move a node's overlay key. +type Rekey struct { + // Previous is the overlay key the node holds now, as the mesh records it. A rekey naming + // another is stale — a replay, or made against a record that moved on — and is refused. + Previous string `json:"previous"` + OverlayKey string `json:"overlay_key"` + Tunnel *Tunnel `json:"tunnel"` + Proof []byte `json:"proof"` +} + +// RekeyProof is what a node signs when it rekeys: the node, the key it leaves, the key it takes +// and the tunnel it took it from, so a proof cannot be moved to another node or another tunnel. +func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { + var t Tunnel + if tunnel != nil { + t = *tunnel + } + peers := make([]string, 0, len(t.Peers)) + for _, p := range t.Peers { + peers = append(peers, p.PublicKey+"@"+p.Address) + } + return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) } // Held is one file or container found on an adopted node and kept as it was. diff --git a/internal/link/rekey_test.go b/internal/link/rekey_test.go new file mode 100644 index 0000000..95d5cc7 --- /dev/null +++ b/internal/link/rekey_test.go @@ -0,0 +1,135 @@ +package link_test + +import ( + "crypto/ed25519" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq ADR 0105: a hub that enrolled before the mesh knew to take a tunnel over rekeys onto the +// found tunnel's key without re-enrolling — which would rotate every key it holds and remake every +// credential the mesh sealed to it. The rekey rides in a report and is signed with the node's +// identity key; the mesh verifies it against the key it recorded, and refuses one signed by +// another key or one already applied. + +const ( + ownKey = "THE-MESHS-OWN-KEY=======================" + tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key=" +) + +func theTunnel() *link.Tunnel { + return &link.Tunnel{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", + Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey, + Peers: []link.TunnelPeer{{PublicKey: "PEER-A=", Address: "192.0.2.2/32"}}} +} + +// anEnrolledHub is a hub the way it stands before the feature: adopted, placed, its overlay key its +// own, its identity key recorded — and a mesh holding both stores. +func anEnrolledHub(t *testing.T) (link.Enrolment, inventory.Node, ed25519.PrivateKey) { + t.Helper() + inv := inventory.ForTest(t) + ident := identity.ForTest(t) + ctx := t.Context() + hub, err := inv.AddNodeAs(ctx, "anchor", true) + if err != nil { + t.Fatal(err) + } + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + if _, err := ident.RecordNodeKey(ctx, hub.ID, public); err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(ctx, hub.ID, ownKey); err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(ctx, "anchor", "anchor.example:51900", "hosting", true, "10.42.0.1"); err != nil { + t.Fatal(err) + } + return link.Enrolment{Inventory: inv, Identity: ident}, hub, private +} + +func TestASignedRekeyMovesTheHubOntoItsTunnel(t *testing.T) { + e, hub, private := anEnrolledHub(t) + ctx := t.Context() + rekey := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()} + rekey.Proof = ed25519.Sign(private, link.RekeyProof("anchor", ownKey, tunnelKey, theTunnel())) + + if err := e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey}); err != nil { + t.Fatal(err) + } + placed, err := e.Inventory.Overlays(ctx) + if err != nil || len(placed) != 1 { + t.Fatal(placed, err) + } + if placed[0].Key != tunnelKey || placed[0].Address != "192.0.2.1" { + t.Fatalf("the hub is not on the tunnel's key and address: %+v", placed[0]) + } + tunnel, _, adopted, err := e.Inventory.AdoptedTunnel(ctx) + if err != nil || !adopted || tunnel.Range != "192.0.2.0/24" || len(tunnel.Peers) != 1 { + t.Fatalf("the tunnel is not adopted after the rekey: %+v %t %v", tunnel, adopted, err) + } + _ = hub + + // Replayed, it is stale: the previous key it names is no longer the node's. + err = e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey}) + if err == nil || !strings.Contains(err.Error(), "previous overlay key") { + t.Fatalf("a replayed rekey was accepted: %v", err) + } +} + +func TestARekeySignedByAnotherKeyIsRefusedAndChangesNothing(t *testing.T) { + e, _, _ := anEnrolledHub(t) + ctx := t.Context() + _, stranger, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + rekey := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()} + rekey.Proof = ed25519.Sign(stranger, link.RekeyProof("anchor", ownKey, tunnelKey, theTunnel())) + + err = e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey}) + if err == nil || !strings.Contains(err.Error(), "not signed by anchor's identity key") { + t.Fatalf("a rekey signed by a stranger was accepted: %v", err) + } + placed, _ := e.Inventory.Overlays(ctx) + if placed[0].Key != ownKey || placed[0].Address != "10.42.0.1" { + t.Fatalf("a refused rekey changed the record: %+v", placed[0]) + } + if _, _, adopted, _ := e.Inventory.AdoptedTunnel(ctx); adopted { + t.Fatal("a refused rekey recorded a tunnel") + } + + // And a proof moved to another tunnel — the signature was over one tunnel, the message names + // another — does not verify either. + moved := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()} + other := theTunnel() + other.Port = 51820 + moved.Proof = ed25519.Sign(mustPrivate(t, e, "anchor"), link.RekeyProof("anchor", ownKey, tunnelKey, other)) + if err := e.Heard(ctx, link.Report{Node: "anchor", Rekey: moved}); err == nil { + t.Fatal("a proof over another tunnel was accepted") + } +} + +// mustPrivate is a fresh key recorded as the node's live one, for signing in a test that needs +// the node's own signature after the fixture's key is out of scope. +func mustPrivate(t *testing.T, e link.Enrolment, node string) ed25519.PrivateKey { + t.Helper() + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + n, err := e.Inventory.NodeByName(t.Context(), node) + if err != nil { + t.Fatal(err) + } + if _, err := e.Identity.RecordNodeKey(t.Context(), n.ID, public); err != nil { + t.Fatal(err) + } + return private +} diff --git a/internal/overlay/declaration.go b/internal/overlay/declaration.go index 0039fb4..65b4e30 100644 --- a/internal/overlay/declaration.go +++ b/internal/overlay/declaration.go @@ -43,6 +43,40 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) { keyPath = DefaultKeyPath } + up := Resource{ + "id": "overlay-up", "type": "service", "unit": Unit, + "state": "running", + // Enabled, so the node comes back onto the network after a reboot without waiting to + // be told again. A node whose overlay only exists while something is watching is not + // a node that survives being switched off and on. + "boot": "enabled", + // And restarted when the peer list changes, because a running interface does not + // re-read its configuration. + // + // This is the whole of it: a node joins, every existing node's peer list changes, + // each file is replaced — and without this the service is already running, nothing + // reloads it, and every node keeps a network that no longer matches the mesh. It + // reports complete success. The lab found it the moment a third node arrived. + // + // Declared state rather than a command. The service must reflect the file; the host + // works out that it does not. A command to restart would be an action, and the link + // may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly, + // which is how this shape was arrived at. + "restart-on": []string{"overlay-config"}, + } + if node.TakesOver != nil { + // The private network takes over the tunnel it found (novox/hq ADR 0105): before this + // unit starts, the host stops and disables the found one — never flushing it — and keeps + // its configuration like any held file. The key is already the found one: the node took + // it as its own overlay key when it enrolled, which is why the mesh's peer list for it + // carries the found peers under the key they know. + up["takes-over"] = map[string]any{ + "interface": node.TakesOver.Interface, + "unit": node.TakesOver.Unit, + "config": node.TakesOver.Config, + } + } + resources := []Resource{ { "id": "overlay-tools", "type": "package", "package": "wireguard-tools", @@ -55,27 +89,7 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) { "mode": "0600", "content": config(node, peers, keyPath), }, - { - "id": "overlay-up", "type": "service", "unit": Unit, - "state": "running", - // Enabled, so the node comes back onto the network after a reboot without waiting to - // be told again. A node whose overlay only exists while something is watching is not - // a node that survives being switched off and on. - "boot": "enabled", - // And restarted when the peer list changes, because a running interface does not - // re-read its configuration. - // - // This is the whole of it: a node joins, every existing node's peer list changes, - // each file is replaced — and without this the service is already running, nothing - // reloads it, and every node keeps a network that no longer matches the mesh. It - // reports complete success. The lab found it the moment a third node arrived. - // - // Declared state rather than a command. The service must reflect the file; the host - // works out that it does not. A command to restart would be an action, and the link - // may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly, - // which is how this shape was arrived at. - "restart-on": []string{"overlay-config"}, - }, + up, } // The names used to be appended here, on the argument that a node with peers and no names is diff --git a/internal/overlay/declaration_test.go b/internal/overlay/declaration_test.go index 10ccdc6..8a941ff 100644 --- a/internal/overlay/declaration_test.go +++ b/internal/overlay/declaration_test.go @@ -223,3 +223,40 @@ func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) { "compromised one could do") } } + +// novox/hq ADR 0105: a node whose private network takes over the tunnel it found is told so on +// the interface's service, and nothing else about the declaration changes — the key is already +// the found one, taken at enrolment. +func TestTakingOverAFoundTunnelIsSaidOnTheInterfacesService(t *testing.T) { + node := Node{Name: "anchor", Key: "PUB", Address: "192.0.2.1", Hub: true, + Endpoint: "198.51.100.1:51900", + TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf"}} + config, resources := declarationFor(t, node, nil) + + var up map[string]any + for _, r := range resources { + if r["type"] == "service" { + up = r + } + } + takes, ok := up["takes-over"].(map[string]any) + if !ok { + t.Fatalf("the interface's service does not say what it takes over: %+v", up) + } + if takes["unit"] != "wg-quick@wg0" || takes["config"] != "/etc/wireguard/wg0.conf" || takes["interface"] != "wg0" { + t.Errorf("the takeover names the wrong tunnel: %+v", takes) + } + if !strings.Contains(config, "ListenPort = 51900") { + t.Errorf("the hub's interface does not listen on the tunnel's port:\n%s", config) + } + if strings.Contains(config, "PrivateKey") { + t.Error("the found key travelled in the configuration; it is the node's own, set from its key file") + } + + _, plain := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "192.0.2.4"}, nil) + for _, r := range plain { + if _, says := r["takes-over"]; says { + t.Error("a node taking over nothing was told to take something over") + } + } +} diff --git a/internal/overlay/graph.go b/internal/overlay/graph.go index 0f1fa01..c4f9c28 100644 --- a/internal/overlay/graph.go +++ b/internal/overlay/graph.go @@ -26,6 +26,48 @@ type Node struct { Site string Hub bool Address string + + // Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the + // mesh has no record of, each known by the public key and the address the found tunnel routed + // to it. Only a hub has any. They stay in its peer list until a node enrols with that key — + // from then on the node is the peer. + Carried []Carried + // TakesOver names the found tunnel this node's private network replaces: its unit is stopped + // and disabled, never flushed, and its configuration kept, before the mesh's interface comes + // up with the found key. Nil on a node that raises the mesh's interface beside whatever it has. + TakesOver *TakeOver +} + +// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a +// node of the mesh. +type Carried struct { + Key string + Address string +} + +// TakeOver is the found tunnel a node's private network takes over, as the host is told it. +type TakeOver struct { + Interface string + Unit string + Config string +} + +// HostPrefix is one address as a route: /32 for IPv4, /128 for IPv6. +func HostPrefix(address string) string { + if strings.Contains(address, ":") { + return address + "/128" + } + return address + "/32" +} + +// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named +// by the key its packets arrive under. +func CarriedName(key string) string { + short := key + if len(short) > 8 { + short = short[:8] + "…" + } + return "a peer of the tunnel (" + short + ")" } // Reachable reports whether other nodes can dial this one. Declared, never inferred. @@ -145,7 +187,9 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) { // The hub holds every node that does not share a site with it, because those nodes // route through it and it must know where to send the replies. Ones it cannot dial // will dial it. + enrolled := map[string]bool{} for _, other := range usable { + enrolled[other.Key] = true if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) { continue } @@ -156,6 +200,21 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) { Why: "routes through this hub", }) } + // And every peer of the tunnel it took over that has not enrolled (novox/hq ADR + // 0105): the same key and the same address the found tunnel had for it, so the + // machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in, + // as it always did. Once a node enrols with that key, the node's entry above is the + // peer, and WireGuard takes one entry per key. + for _, c := range self.Carried { + if enrolled[c.Key] { + continue + } + peers = append(peers, Peer{ + Name: CarriedName(c.Key), Key: c.Key, + Allowed: HostPrefix(c.Address), + Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh", + }) + } } sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name }) diff --git a/internal/overlay/graph_test.go b/internal/overlay/graph_test.go index 14b2592..5cfe6ea 100644 --- a/internal/overlay/graph_test.go +++ b/internal/overlay/graph_test.go @@ -304,3 +304,39 @@ func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) { "nowhere to go") } } + +// novox/hq ADR 0105: a hub that took over the predecessor's tunnel carries every peer that tunnel +// had, under the key and at the address the peer knows, until a node enrols with that key. +func TestTheHubCarriesTheTunnelsPeersUntilTheyEnrol(t *testing.T) { + hub := at("anchor", "hosting", "192.0.2.1", "198.51.100.1:51900", true) + hub.Carried = []Carried{ + {Key: "key-home", Address: "192.0.2.2"}, + {Key: "key-workstation", Address: "192.0.2.3"}, + } + // The machine behind key-home enrolled: it is a node now, at the address it kept. + home := at("home", "house", "192.0.2.2", "", false) + home.Key = "key-home" + + g, err := Compute([]Node{hub, home}, "192.0.2.0/24") + if err != nil { + t.Fatal(err) + } + peers := peersOf(t, g, "anchor") + carried, ok := peers[CarriedName("key-workstation")] + if !ok { + t.Fatalf("the hub does not carry the peer that has not enrolled: %+v", g["anchor"]) + } + if carried.Allowed != "192.0.2.3/32" || carried.Endpoint != "" || carried.Key != "key-workstation" { + t.Errorf("a carried peer is not the tunnel's own entry — same key, its one address, no endpoint: %+v", carried) + } + if _, twice := peers[CarriedName("key-home")]; twice { + t.Error("a peer that enrolled is carried as well as listed as a node: WireGuard takes one entry per key") + } + if node, ok := peers["home"]; !ok || node.Key != "key-home" || node.Allowed != "192.0.2.2/32" { + t.Errorf("the enrolled peer is not the node it became: %+v", node) + } + // Carried peers are the hub's business only: a spoke routes everything through the hub. + if _, leaked := peersOf(t, g, "home")[CarriedName("key-workstation")]; leaked { + t.Error("a carried peer appeared in a spoke's peer list") + } +} diff --git a/lab/adopt-the-tunnel/README.md b/lab/adopt-the-tunnel/README.md new file mode 100644 index 0000000..15d9ae6 --- /dev/null +++ b/lab/adopt-the-tunnel/README.md @@ -0,0 +1,109 @@ +# Lab bed: the hub adopts the predecessor's tunnel (novox/hq ADR 0105) + +A scenario and an integration-test skeleton for the mesh-lab repository, kept here because this +branch changes only the controller and the host. Move `adopt-the-tunnel.yml` to +`mesh-lab/scenarios/` and `adopt-the-tunnel.test.ts` to `mesh-lab/test/integration/` when the +feature lands; neither has been run. The skeleton follows `adoption.test.ts` and reuses its +harness. Documentation addresses throughout; the bed is node-agnostic. + +## The bed, precisely + +Three machines on one public segment, `hosting` (192.0.2.0/24), inbound allowed on all (the +anchor's firewall is the predecessor's, installed by the bed): + +| machine | address | role | +|---|---|---| +| `anchor` | 192.0.2.10 | the machine in use: the predecessor's hub, then the mesh adopted on it | +| `peer-a` | 192.0.2.20 | a predecessor machine: reaches a service on the anchor through the tunnel; later **enrols and keeps its address** | +| `peer-b` | 192.0.2.30 | a second predecessor machine: reaches the same service; **never enrols** — the peer that must notice nothing throughout | +| `fresh` | 192.0.2.40 | a new machine: enrols later and **gets a fresh address from the same range** | + +**Prepared the way the predecessor leaves a hub** (before genesis, by the bed, on `anchor`): + +- `wireguard-tools` installed; a keypair made on each of `anchor`, `peer-a`, `peer-b`. +- `/etc/wireguard/wg0.conf` on the anchor: `[Interface]` `PrivateKey = `, + `ListenPort = 51900`, `Address = 10.10.0.1/24`; two `[Peer]` sections — `peer-a`'s public + key with `AllowedIPs = 10.10.0.2/32`, `peer-b`'s with `AllowedIPs = 10.10.0.3/32`. Raised with + `systemctl enable --now wg-quick@wg0`. **10.10.0.0/24 is deliberately not the mesh's default + range** (10.42.0.0/16), so a hub address in 10.10.0.0/24 can only have come from the tunnel. +- `wg0.conf` on each peer: its own key, `Address = 10.10.0.2/24` (resp. `.3/24`), one + `[Peer]` — the anchor's public key, `Endpoint = 192.0.2.10:51900`, + `AllowedIPs = 10.10.0.0/24`, `PersistentKeepalive = 25`. Raised the same way. +- A service on the anchor the peers reach **only over the tunnel**: a container publishing + `10.10.0.1:8081:80` (bound to the tunnel address, so a call from 192.0.2.20 to 10.10.0.1:8081 + proves the tunnel carried it). Under a name no catalogue module uses — this bed is about the + tunnel, not about taking a service. +- The predecessor's firewall (`ufw`) allowing `51900/udp` and `22/tcp`, denying the rest — as ADR + 0100's bed prepares it. +- A record of the anchor's `wg0` public key and of `sha256sum /etc/wireguard/wg0.conf`, taken + before genesis, for the assertions below. + +**Genesis**, adopted, on the anchor: `mesh-bootstrap --adopted --node anchor --site hosting +--endpoint 192.0.2.10:51900 …` — no `--hub-port`, no `--overlay-range`, no `--tunnel`: the +installer finds `wg0` itself (one interface besides `mesh0`) and takes its port and range. The +bed asserts genesis **says** it found and took the tunnel. + +**Review changes (2026-09-24).** A spoke's `wg0.conf` names one peer — the hub — routed the +whole range; the controller skips range-routed peers, so T2's enrolment carries no peer from the +spoke. A hub that enrolled *before* this feature (a generated key) takes the tunnel over without +re-enrolling: `mesh-host overlay take --tunnel wg0` on the machine rekeys the overlay key only and +sends a signed rekey; the bed adds **R0** for it below. A takeover is composed only for a hub +placed at the tunnel's address on the tunnel's port, and the host stops nothing until the declared +interface matches the found one and the key file holds the found key; a mesh interface that fails +to start gives the found unit back. The host's account has three states: `not-taken`, `taken`, +`down`. + +## Assertions, in the record's order + +- **R0 — a hub enrolled with its own key takes the tunnel over by rekeying.** Genesis is run + adopted *without* the tunnel being found (the bed stops `wg-quick@wg0` for the run, so the + installer sees no tunnel, then starts it again — the pre-feature shape). Then on the anchor: + `mesh-host overlay take --tunnel wg0`; `node show anchor` says "tunnel found wg0 …"; `overlay + place anchor --hub --endpoint 192.0.2.10:51900 --site hosting` (with `:51820` first, which must + be refused naming 51900); `plan anchor --json` names `Address = 10.10.0.1/32`, `ListenPort = + 51900`, two `/32` peers, `takes-over` wg0 and nothing in 10.42.0.0/16; then `push anchor --wait + 2m` and T1's assertions hold. `overlay take` run a second time is refused by the controller as + stale and changes nothing. + +- **T1 — the tunnel changes hands and the peers notice nothing.** After genesis and the push + that raises the private network on the anchor: + - `wg show interfaces` on the anchor lists `mesh0` and not `wg0`; + `systemctl is-active wg-quick@wg0` is inactive and `is-enabled` disabled; + - `/etc/wireguard/wg0.conf` is on disk with the recorded digest (kept, never flushed), and + `node show anchor` names where its original was kept; + - `wg show mesh0 public-key` is the anchor's recorded `wg0` public key; `wg show mesh0 + listen-port` is 51900; `ip -o addr show dev mesh0` carries `10.10.0.1`; `wg show mesh0 peers` + lists both peers' public keys with their `/32` allowed addresses; + - a loop on `peer-a` and `peer-b` calling `http://10.10.0.1:8081/` every second, started before + genesis, records **no window of failure longer than one WireGuard re-handshake** (measure and + assert an upper bound — the switch is one unit stop plus one unit start on the anchor); the + peers' `wg0.conf` digests are unchanged; the peers' `wg show wg0 latest-handshakes` advance + after the switch. + - `overlay show` lists `anchor` as the hub over the tunnel it took over, and both peers under + "peers of the tunnel … not nodes of the mesh", not yet enrolled. +- **T2 — a peer enrols and keeps its address.** On `peer-a`: `node add peer-a --adopted`, + token issued, `mesh-host enrol --token …` **with the broker reached over the tunnel** (the + broker address in the token is `10.10.0.1:`, which only the tunnel routes); then `overlay + place peer-a --site house` and a push. Assert: `node show peer-a` says a tunnel `wg0` was + found and `overlay show` puts `peer-a` at **10.10.0.2**; the carried-peers list now says + `enrolled as peer-a`; the anchor's `mesh0` still has exactly one entry for `peer-a`'s key; + `peer-a`'s `wg0` is down and `mesh0` up with the same key; `peer-a` still reaches + `10.10.0.1:8081` and `peer-b` still does too, uninterrupted. +- **T3 — a new machine gets a fresh address from the same range.** `fresh` enrols converged + (no tunnel), is placed, pushed. Assert its address is **10.10.0.4** (`.1` hub, `.2` and `.3` + the tunnel's), that it reaches `10.10.0.1` (the hub) and `10.10.0.2` (the enrolled peer) — + `ping -c1` over `mesh0` — and that `peer-b`, never enrolled, is still served. +- **T4 — nothing derived from the address is stale.** `plan anchor --json` and `plan peer-a + --json` (and the rendered `/etc/hosts` on each node) name `10.10.0.1` for the anchor and + `10.10.0.2` for `peer-a`, and no address in `10.42.0.0/16`; the broker address handed to a + module issued on `peer-a` is `anchor.internal:` resolving to `10.10.0.1`; the same after a + second `push` of every node, byte for byte. +- **N — the narrowed ADR 0100 check.** On `fresh`, a converged genesis dry-run with + `--overlay-range 10.10.0.0/24` while a *second* tunnel the bed raises there (`wg1` at + 10.10.0.9/24, not adopted because the node is converged) is up, is refused naming `wg1` — + the non-overlap rule still applies where a tunnel is not adopted. + +## What is not asserted here + +- Taking a service over the tunnel (ADR 0100's bed does that). +- IPv6 tunnels: the parser reads them, the bed prepares only IPv4. diff --git a/lab/adopt-the-tunnel/adopt-the-tunnel.test.ts b/lab/adopt-the-tunnel/adopt-the-tunnel.test.ts new file mode 100644 index 0000000..7a3f479 --- /dev/null +++ b/lab/adopt-the-tunnel/adopt-the-tunnel.test.ts @@ -0,0 +1,174 @@ +/** + * THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). Skeleton — NOT YET RUN. + * + * The bed and every assertion are described in README.md beside this file; the numbered + * assertions here are that document's. Follows adoption.test.ts: same harness, same `on`/`must` + * helpers, same genesis wrapper. + * + * MESH_LAB_INCUS='sudo -n incus' + * MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host + * MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock + * MESH_LAB_CATALOG=.../mesh-catalog/modules + */ +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, catalogueIsPresent } from "./harness.ts"; +import { genesis } from "./genesis.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; +const skip = !capability.usable ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle" + : catalogueIsPresent(); + +const SCENARIO = "adopt-the-tunnel"; +const ANCHOR = "anchor", PEER_A = "peer-a", PEER_B = "peer-b", FRESH = "fresh"; +const TUNNEL = { port: 51900, range: "10.10.0.0/24", hub: "10.10.0.1", a: "10.10.0.2", b: "10.10.0.3", fresh: "10.10.0.4" }; +const SERVICE = `http://${TUNNEL.hub}:8081/`; + +let instanceId = ""; +let wg0Key = ""; // the anchor's wg0 public key, recorded before genesis +let wg0Digest = ""; // sha256 of /etc/wireguard/wg0.conf before genesis + +async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} +async function must(machine: string, command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(machine, command, timeoutMs); + if (!ok) throw new Error(`${machine}: ${command}\n${out}`); + return out; +} +/** The controller, a container on the anchor. */ +async function control(args: string): Promise { + return must(ANCHOR, `docker exec mesh-controller mesh-controller ${args}`); +} + +/** Prepares a machine the way the predecessor leaves it: a keypair and a wg0 — see README.md. */ +async function predecessorTunnelOn(machine: string, conf: (keys: Record) => string, keys: Record): Promise { + await must(machine, "apt-get install -y wireguard-tools >/dev/null 2>&1 || pacman -S --noconfirm wireguard-tools >/dev/null"); + await must(machine, `umask 077; printf '%s' '${conf(keys)}' > /etc/wireguard/wg0.conf`); + await must(machine, "systemctl enable --now wg-quick@wg0"); +} + +before(async () => { + if (skip) return; + await destroyAll(SCENARIO); + const scenario = loadScenario(`scenarios/${SCENARIO}.yml`); + instanceId = (await raise(scenario)).instanceId; + + // Keys for the three predecessor machines, made where they live and never moved. + const keys: Record = {}; + for (const m of [ANCHOR, PEER_A, PEER_B]) { + await must(m, "umask 077; wg genkey > /etc/wireguard/predecessor.key"); + keys[m] = (await must(m, "wg pubkey < /etc/wireguard/predecessor.key")).trim(); + } + await predecessorTunnelOn(ANCHOR, k => [ + "[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `ListenPort = ${TUNNEL.port}`, `Address = ${TUNNEL.hub}/24`, + "[Peer]", `PublicKey = ${k[PEER_A]}`, `AllowedIPs = ${TUNNEL.a}/32`, + "[Peer]", `PublicKey = ${k[PEER_B]}`, `AllowedIPs = ${TUNNEL.b}/32`, + ].join("\n"), keys); + for (const [m, addr] of [[PEER_A, TUNNEL.a], [PEER_B, TUNNEL.b]] as const) { + await predecessorTunnelOn(m, k => [ + "[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `Address = ${addr}/24`, + "[Peer]", `PublicKey = ${k[ANCHOR]}`, `Endpoint = 192.0.2.10:${TUNNEL.port}`, `AllowedIPs = ${TUNNEL.range}`, "PersistentKeepalive = 25", + ].join("\n"), keys); + } + // A service reachable only over the tunnel, under a name no catalogue module uses. + await must(ANCHOR, `docker run -d --name predecessor-page -p ${TUNNEL.hub}:8081:80 nginx:alpine`); + // The predecessor's firewall: the tunnel's port and ssh, nothing else (as ADR 0100's bed). + await must(ANCHOR, `ufw --force reset >/dev/null; ufw default deny incoming; ufw allow 22/tcp; ufw allow ${TUNNEL.port}/udp; ufw --force enable`); + for (const m of [PEER_A, PEER_B]) assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service over the tunnel before genesis`); + + wg0Key = (await must(ANCHOR, "wg show wg0 public-key")).trim(); + wg0Digest = (await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0]; + // The probe the peers keep running through the switch: one call a second, failures counted. + for (const m of [PEER_A, PEER_B]) await must(m, `nohup sh -c 'while :; do curl -fsS --max-time 1 ${SERVICE} >/dev/null 2>&1 || date +%s >> /tmp/failed; sleep 1; done' >/dev/null 2>&1 &`); +}); + +after(async () => { if (instanceId) await destroy(instanceId); }); + +test("T1 — adopted, the tunnel changes hands and the peers notice nothing", { skip }, async () => { + const ran = await genesis({ instanceId, machine: ANCHOR, node: ANCHOR, site: "hosting", + flags: ["--adopted", "--endpoint", `192.0.2.10:${TUNNEL.port}`] } as never); + assert.match(ran.said, /tunnel\s+wg0/i, `genesis did not say it found and took the tunnel:\n${ran.said}`); + + const ifaces = await must(ANCHOR, "wg show interfaces"); + assert.match(ifaces, /\bmesh0\b/); assert.doesNotMatch(ifaces, /\bwg0\b/); + assert.equal((await on(ANCHOR, "systemctl is-active wg-quick@wg0")).out.trim(), "inactive"); + assert.equal((await on(ANCHOR, "systemctl is-enabled wg-quick@wg0")).out.trim(), "disabled"); + assert.equal((await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0], wg0Digest, "the found configuration was changed or flushed"); + assert.equal((await must(ANCHOR, "wg show mesh0 public-key")).trim(), wg0Key, "the mesh's interface is not up with the found key"); + assert.equal((await must(ANCHOR, "wg show mesh0 listen-port")).trim(), String(TUNNEL.port)); + assert.match(await must(ANCHOR, "ip -o addr show dev mesh0"), new RegExp(TUNNEL.hub.replaceAll(".", "\\."))); + const peers = await must(ANCHOR, "wg show mesh0 allowed-ips"); + assert.match(peers, new RegExp(`${TUNNEL.a}/32`)); assert.match(peers, new RegExp(`${TUNNEL.b}/32`)); + + for (const m of [PEER_A, PEER_B]) { + const failed = (await on(m, "cat /tmp/failed 2>/dev/null | wc -l")).out.trim(); + assert.ok(Number(failed) <= 5, `${m} lost the service for ${failed} seconds through the switch`); + assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service after the switch`); + } + const shown = await control("overlay show"); + assert.match(shown, /anchor.*hub.*took over on wg0/); + assert.match(shown, /peers of the tunnel/); assert.match(shown, /not yet enrolled/); + assert.match(await control(`node show ${ANCHOR}`), /tunnel found\s+wg0 on port 51900/); +}); + +test("T2 — a peer enrols over the tunnel and keeps its address", { skip }, async () => { + await control(`node add ${PEER_A} --adopted`); + const token = (await control(`token issue --node ${PEER_A}`)).match(/token\s+(\S+)/)?.[1] ?? ""; + // The token's broker address is the hub's tunnel address: only the tunnel routes it. + await must(PEER_A, `mesh-host enrol --token '${token}'`); + await control(`overlay place ${PEER_A} --site house`); + await control(`assign ${PEER_A} networking`); + await control(`push ${PEER_A} --wait 2m`); + + assert.match(await control("overlay show"), new RegExp(`${PEER_A}\\s+${TUNNEL.a.replaceAll(".", "\\.")}`)); + assert.match(await control("overlay show"), new RegExp(`enrolled as ${PEER_A}`)); + const onHub = await must(ANCHOR, "wg show mesh0 allowed-ips"); + assert.equal(onHub.split("\n").filter(l => l.includes(`${TUNNEL.a}/32`)).length, 1, "the enrolled peer's key appears twice on the hub"); + assert.doesNotMatch(await must(PEER_A, "wg show interfaces"), /\bwg0\b/); + assert.ok((await on(PEER_A, `curl -fsS --max-time 3 ${SERVICE}`)).ok); + assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok, "the peer that never enrols lost the service"); +}); + +test("T3 — a new machine gets a fresh address from the same range", { skip }, async () => { + await control(`node add ${FRESH}`); + const token = (await control(`token issue --node ${FRESH}`)).match(/token\s+(\S+)/)?.[1] ?? ""; + await must(FRESH, `mesh-host enrol --token '${token}'`); + await control(`overlay place ${FRESH} --nothing`); + await control(`assign ${FRESH} networking`); + await control(`push ${FRESH} --wait 2m`); + assert.match(await control("overlay show"), new RegExp(`${FRESH}\\s+${TUNNEL.fresh.replaceAll(".", "\\.")}`)); + assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.hub}`)).ok, "the new machine does not reach the hub"); + assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.a}`)).ok, "the new machine does not reach the enrolled peer"); + assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok); +}); + +test("T4 — nothing derived from the address is stale", { skip }, async () => { + for (const n of [ANCHOR, PEER_A, FRESH]) { + const plan = await control(`plan ${n} --json`); + assert.doesNotMatch(plan, /10\.42\./, `${n}'s plan names the mesh's default range`); + assert.match(plan, new RegExp(TUNNEL.hub.replaceAll(".", "\\."))); + assert.match(await must(n, "cat /etc/hosts"), new RegExp(`${TUNNEL.hub.replaceAll(".", "\\.")}\\s+anchor\\.internal`)); + } + const first = await control(`plan ${PEER_A} --json`); + await control("push"); + assert.equal(await control(`plan ${PEER_A} --json`), first, "a push changed what the plan says"); +}); + +test("N — where a tunnel is not adopted, the ranges must still differ (ADR 0100)", { skip }, async () => { + await must(FRESH, "umask 077; printf '[Interface]\\nPrivateKey = %s\\nAddress = 10.10.0.9/24\\n' \"$(wg genkey)\" > /etc/wireguard/wg1.conf; systemctl start wg-quick@wg1"); + const ran = await genesis({ instanceId, machine: FRESH, node: FRESH, flags: ["--dry-run", "--overlay-range", TUNNEL.range], attempts: 1, verify: false, hostService: false } as never); + assert.match(ran.said, /wg1/, `a converged genesis did not refuse the overlapping tunnel it does not adopt:\n${ran.said}`); +}); diff --git a/lab/adopt-the-tunnel/adopt-the-tunnel.yml b/lab/adopt-the-tunnel/adopt-the-tunnel.yml new file mode 100644 index 0000000..ba1b3bc --- /dev/null +++ b/lab/adopt-the-tunnel/adopt-the-tunnel.yml @@ -0,0 +1,50 @@ +# THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). See README.md beside this file. +# +# hosting (public) +# anchor 192.0.2.10 the predecessor's hub: wg0 on udp/51900, 10.10.0.1/24, two peers; then the +# mesh adopted on it, taking the tunnel over +# peer-a 192.0.2.20 a predecessor machine reaching a service on the anchor over the tunnel; +# enrols later and keeps 10.10.0.2 +# peer-b 192.0.2.30 a predecessor machine that never enrols: must notice nothing, ever +# fresh 192.0.2.40 a new machine: enrols later and gets 10.10.0.4 +# +# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the bed. +scenario: adopt-the-tunnel + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true + inbound: allow + memory: 12GiB + cpus: 6 + disk: 60GiB + peer-a: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 20GiB + peer-b: + at: { segment: hosting, address: [192.0.2.30] } + egress: true + inbound: allow + memory: 2GiB + cpus: 2 + disk: 15GiB + fresh: + at: { segment: hosting, address: [192.0.2.40] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 20GiB + +place: + all: [host, runtime]