Adopt the predecessor's tunnel in place: its range, its address, its peers
On an adopted hub the private network takes over the tunnel it finds rather than running beside it (hq ADR 0105): two tunnels leave the mesh's unreachable through the provider's filter, so no machine can ever join. The node presents the found tunnel when it enrols, under the key it took as its own; the inventory records it (node.tunnel, tunnel_peer — migration 0031) and the mesh composes from it: the overlay's range is the adopted tunnel's, the hub is placed at the tunnel's address on the tunnel's port, and every peer the tunnel had is carried in the hub's peer list as a peer of the tunnel, not a node of the mesh, until a node enrols with that key — which then keeps the address the tunnel had for it. A fresh node never gets an address the tunnel holds. The hub's declaration tells the host which unit to take over; the host's account of carrying it is recorded and shown. Every reader of the range follows the setting; nothing stores it. A found tunnel under another key is recorded and not adopted, so ADR 0100's non-overlap rule keeps applying where a tunnel is left running beside the mesh's. A lab bed and test skeleton for "How it is checked" are under lab/.
This commit is contained in:
@@ -45,6 +45,9 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
|
||||
if err := showTunnel(ctx, inv, node.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(said.Held) == 0 {
|
||||
fmt.Printf(" holding nothing found\n")
|
||||
}
|
||||
@@ -63,6 +66,37 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
return nil
|
||||
}
|
||||
|
||||
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
|
||||
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
|
||||
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
tunnel, err := inv.TunnelOf(ctx, name)
|
||||
if errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
|
||||
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
|
||||
carried, said, err := inv.CarriedTunnelOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case !said:
|
||||
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
|
||||
case carried.Taken:
|
||||
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
|
||||
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
|
||||
default:
|
||||
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
|
||||
}
|
||||
if said && carried.Kept != "" {
|
||||
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if s == "" {
|
||||
return "none reported"
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,11 +22,28 @@ import (
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
func overlayCIDR() string {
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v
|
||||
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
|
||||
const DefaultOverlayCIDR = "10.42.0.0/16"
|
||||
|
||||
// overlayRange is the range the mesh allocates node addresses from.
|
||||
//
|
||||
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
|
||||
// found is at that tunnel's address, its peers are at theirs, and every node's address is
|
||||
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
|
||||
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
|
||||
// the range genesis was told, or the default.
|
||||
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "10.42.0.0/16"
|
||||
if adopted {
|
||||
return tunnel.Range, nil
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
return DefaultOverlayCIDR, nil
|
||||
}
|
||||
|
||||
func overlayCommand(ctx context.Context, args []string) error {
|
||||
@@ -110,16 +128,46 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
|
||||
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
|
||||
// have the mesh's interface up where no peer is listening for it.
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tunnel inventory.Tunnel
|
||||
adoptsTunnel := false
|
||||
if *hub && found.Adopted {
|
||||
if t, err := inv.TunnelOf(ctx, node); err == nil {
|
||||
tunnel = t
|
||||
placed, _ := inv.Overlays(ctx)
|
||||
for _, o := range placed {
|
||||
if o.Name == node && o.Key == t.PublicKey {
|
||||
adoptsTunnel = true
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if adoptsTunnel {
|
||||
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
|
||||
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
|
||||
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
|
||||
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
||||
// election by address prefix fails silently (novox/hq ADR 0007).
|
||||
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
|
||||
address, err := inv.AssignAddress(ctx, found.ID, cidr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -128,6 +176,10 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
|
||||
" `module issue` them again and push (novox/hq issue 059)")
|
||||
switch {
|
||||
case adoptsTunnel:
|
||||
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
|
||||
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
|
||||
len(tunnel.Peers))
|
||||
case *hub:
|
||||
fmt.Println(" the hub — every node not sharing a site routes through it")
|
||||
case *endpoint == "":
|
||||
@@ -154,15 +206,33 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
|
||||
tunnels, err := inv.Tunnels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, overlay.Node{
|
||||
n := overlay.Node{
|
||||
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||
})
|
||||
}
|
||||
if t, takes := tunnels[p.Name]; takes {
|
||||
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config}
|
||||
}
|
||||
if p.Hub {
|
||||
for _, c := range carried {
|
||||
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
||||
@@ -170,7 +240,11 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
||||
return overlay.Empty(), nil
|
||||
}
|
||||
g, err := overlay.From(nodes, overlayCIDR(), "")
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
@@ -292,6 +366,17 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
|
||||
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
|
||||
// mesh until they enrol — and once one has, it is listed as the node it became.
|
||||
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
carried, err := open.inventory.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
place := n.Address
|
||||
if place == "" {
|
||||
@@ -301,6 +386,12 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
}
|
||||
fmt.Printf("%-16s %-14s", n.Name, place)
|
||||
switch {
|
||||
case n.Hub && adopted:
|
||||
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's",
|
||||
tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub")
|
||||
case !n.Reachable():
|
||||
@@ -309,14 +400,35 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
if n.Site != "" {
|
||||
fmt.Printf(" at %s", n.Site)
|
||||
}
|
||||
if n.TakesOver != nil && !n.Hub {
|
||||
fmt.Printf(" takes over %s", n.TakesOver.Interface)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, p := range computed[n.Name] {
|
||||
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
||||
}
|
||||
}
|
||||
if len(carried) > 0 {
|
||||
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
|
||||
for _, c := range carried {
|
||||
state := "not yet enrolled"
|
||||
if c.EnrolledAs != "" {
|
||||
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
|
||||
}
|
||||
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// portOfEndpoint is the port in host:port, or empty.
|
||||
func portOfEndpoint(endpoint string) string {
|
||||
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||
return endpoint[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SilentFor is how long a node may be quiet before the mesh says so.
|
||||
//
|
||||
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
||||
|
||||
@@ -108,3 +108,84 @@ func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
|
||||
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
|
||||
// the tunnel the hub holds — never stored anywhere else.
|
||||
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
||||
|
||||
before, err := overlayRange(ctx, inv)
|
||||
if err != nil || before != "10.99.0.0/16" {
|
||||
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
|
||||
}
|
||||
|
||||
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
|
||||
// tunnel's key, and presenting the tunnel.
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
|
||||
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
after, err := overlayRange(ctx, inv)
|
||||
if err != nil || after != "192.0.2.0/24" {
|
||||
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
|
||||
}
|
||||
|
||||
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
|
||||
// the tunnel's port.
|
||||
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
|
||||
if err == nil || !strings.Contains(err.Error(), "51900") {
|
||||
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
|
||||
}
|
||||
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
|
||||
}
|
||||
|
||||
// And the hub's declaration carries the peer and the takeover.
|
||||
nodes, computed, err := graph(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var hubNode overlay.Node
|
||||
for _, n := range nodes {
|
||||
if n.Name == "anchor" {
|
||||
hubNode = n
|
||||
}
|
||||
}
|
||||
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
|
||||
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
|
||||
}
|
||||
carried := false
|
||||
for _, p := range computed["anchor"] {
|
||||
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
|
||||
carried = true
|
||||
}
|
||||
}
|
||||
if !carried {
|
||||
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,25 +16,64 @@ import (
|
||||
// node's peer list to chase it, and an address that moves is the thing declaring the hub was
|
||||
// meant to stop.
|
||||
//
|
||||
// Allocated in order from the range, taking the lowest free one. Not random: a person reading a
|
||||
// peer list should be able to guess which node an address belongs to, and reuse of a released
|
||||
// address is a smaller problem than a list nobody can hold in their head.
|
||||
// **The adopted tunnel's addresses come first** (novox/hq ADR 0105). The hub that took over a
|
||||
// tunnel is at the tunnel's own address. A node enrolling with a key the tunnel already routed
|
||||
// to keeps the address the tunnel had for it — nothing a peer knows changes. And an address the
|
||||
// tunnel holds for a peer that has not enrolled is never handed to anyone else: that peer is
|
||||
// still reaching the hub at it.
|
||||
//
|
||||
// The rest is allocated in order from the range, taking the lowest free one. Not random: a person
|
||||
// reading a peer list should be able to guess which node an address belongs to, and reuse of a
|
||||
// released address is a smaller problem than a list nobody can hold in their head.
|
||||
func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) {
|
||||
prefix, err := netip.ParsePrefix(cidr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err)
|
||||
}
|
||||
|
||||
var existing *string
|
||||
var existing, key *string
|
||||
var name string
|
||||
var hub bool
|
||||
if err := i.store.Pool().QueryRow(ctx,
|
||||
`select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil {
|
||||
`select name, host(overlay_address), overlay_key, is_hub from node where id = $1`, node).
|
||||
Scan(&name, &existing, &key, &hub); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if existing != nil && *existing != "" {
|
||||
return *existing, nil
|
||||
}
|
||||
|
||||
tunnel, hubName, adopted, err := i.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if adopted && hub && hubName == name {
|
||||
address, err := netip.ParsePrefix(tunnel.Address)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the adopted tunnel's address %q: %w", tunnel.Address, err)
|
||||
}
|
||||
return i.place(ctx, node, address.Addr().String())
|
||||
}
|
||||
carried, err := i.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taken := map[string]bool{}
|
||||
if adopted {
|
||||
// The tunnel's own address is the hub's whether or not the hub has been placed yet.
|
||||
if address, err := netip.ParsePrefix(tunnel.Address); err == nil {
|
||||
taken[address.Addr().String()] = true
|
||||
}
|
||||
}
|
||||
for _, p := range carried {
|
||||
if key != nil && p.PublicKey == *key {
|
||||
// The tunnel already routes to this key: the node keeps that address, and the peer
|
||||
// notices nothing when its machine enrols.
|
||||
return i.place(ctx, node, p.Address)
|
||||
}
|
||||
taken[p.Address] = true
|
||||
}
|
||||
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select host(overlay_address) from node where overlay_address is not null`)
|
||||
if err != nil {
|
||||
@@ -58,12 +97,7 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
|
||||
candidate := prefix.Masked().Addr().Next()
|
||||
for prefix.Contains(candidate) {
|
||||
if !taken[candidate.String()] {
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`update node set overlay_address = $2::inet where id = $1`,
|
||||
node, candidate.String()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return candidate.String(), nil
|
||||
return i.place(ctx, node, candidate.String())
|
||||
}
|
||||
candidate = candidate.Next()
|
||||
}
|
||||
@@ -72,5 +106,13 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
|
||||
// halfway through assigning one node.
|
||||
return "", fmt.Errorf(
|
||||
"every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+
|
||||
"range and renumbering it is a deliberate act", cidr, node)
|
||||
"range and renumbering it is a deliberate act", cidr, name)
|
||||
}
|
||||
|
||||
func (i *Inventory) place(ctx context.Context, node, address string) (string, error) {
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`update node set overlay_address = $2::inet where id = $1`, node, address); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return address, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
-- The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
|
||||
--
|
||||
-- On an adopted node that is the hub, the private network takes over the tunnel it finds: its
|
||||
-- key, its port, its address and range, and every peer. The node presents what it found when it
|
||||
-- enrols -- the same moment it presents its keys, because the found tunnel's key IS its key on the
|
||||
-- private network from then on -- and the mesh composes every address from it.
|
||||
|
||||
-- What the node presented: interface, unit and configuration path, port, address and range, and
|
||||
-- the tunnel's public key. The private key never travels; the node keeps it as its own overlay
|
||||
-- key. Null on a node that found no tunnel, which is every converged one.
|
||||
alter table node add column tunnel jsonb;
|
||||
|
||||
-- The node's last account of carrying it: the found interface down and disabled, the mesh's up
|
||||
-- in its place. Null until the node says so.
|
||||
alter table node add column tunnel_carried jsonb;
|
||||
|
||||
-- The peers the found tunnel had: a public key and the address the tunnel routed to it. Peers of
|
||||
-- the tunnel, not nodes of the mesh, until they enrol -- a machine the mesh has no record of, whose
|
||||
-- identity precedes its enrolment. One row per key, and one address per key on one tunnel.
|
||||
create table tunnel_peer (
|
||||
node uuid not null references node(id) on delete cascade,
|
||||
public_key text not null,
|
||||
address inet not null,
|
||||
since timestamptz not null default now(),
|
||||
primary key (node, public_key),
|
||||
unique (node, address)
|
||||
);
|
||||
@@ -0,0 +1,313 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
|
||||
//
|
||||
// On an adopted node that is the hub, the private network takes over the tunnel it finds: its
|
||||
// private key, its port, its address and range, and every peer the found interface had. The node
|
||||
// presents what it found when it enrols, in the same breath as its keys — the found tunnel's key is
|
||||
// its key on the private network from then on — and the mesh composes every address from it: the
|
||||
// hub's is the tunnel's, the range is the tunnel's, and a peer that enrols keeps the address the
|
||||
// tunnel already had for its key.
|
||||
|
||||
// Tunnel is what a node found on its machine, as it presented it. No private key: the node keeps
|
||||
// it as its own overlay key, sealed like any own secret, and the mesh records only the public half
|
||||
// — which is then the node's overlay key too.
|
||||
type Tunnel struct {
|
||||
// Interface, Unit and Config are what the host takes over: the found interface, the unit
|
||||
// that raised it, and its configuration file, which is kept like any held file.
|
||||
Interface string `json:"interface"`
|
||||
Unit string `json:"unit"`
|
||||
Config string `json:"config"`
|
||||
// Port is the port the found interface listened on — one the hosting provider already lets
|
||||
// through, which is why it is worth taking.
|
||||
Port int `json:"port"`
|
||||
// Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the
|
||||
// network that prefix names, 192.0.2.0/24.
|
||||
Address string `json:"address"`
|
||||
Range string `json:"range"`
|
||||
// PublicKey is the found interface's, which every peer knows the tunnel by.
|
||||
PublicKey string `json:"public_key"`
|
||||
// Peers are the found interface's peers: each a public key and the address the tunnel routed
|
||||
// to it.
|
||||
Peers []TunnelPeer `json:"peers,omitempty"`
|
||||
// At is when the node presented it; zero on a tunnel not yet recorded.
|
||||
At time.Time `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// TunnelPeer is one peer of a found tunnel.
|
||||
type TunnelPeer struct {
|
||||
PublicKey string `json:"public_key"`
|
||||
// Address is the one host address the tunnel routed to the peer, without a prefix.
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// Carried is what a node last said about carrying the tunnel it found: the found interface down
|
||||
// and disabled, the mesh's up in its place with the found key.
|
||||
type Carried struct {
|
||||
Interface string `json:"interface"`
|
||||
Port int `json:"port"`
|
||||
Range string `json:"range"`
|
||||
Peers int `json:"peers"`
|
||||
// Taken is whether the found interface is down and the mesh's up with its key; Kept is where
|
||||
// the found configuration's original was kept.
|
||||
Taken bool `json:"taken"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// CarriedPeer is one peer of the adopted tunnel as the mesh holds it: a peer of the tunnel, and
|
||||
// — once a node enrols with that key — a node of the mesh as well.
|
||||
type CarriedPeer struct {
|
||||
PublicKey string
|
||||
Address string
|
||||
// EnrolledAs names the node that enrolled with this key, or is empty while none has.
|
||||
EnrolledAs string
|
||||
}
|
||||
|
||||
// ErrNoTunnel is asking about a tunnel on a node that presented none.
|
||||
var ErrNoTunnel = errors.New("that node presented no tunnel")
|
||||
|
||||
// RecordTunnel keeps what a node presented, replacing what was there: the question is the tunnel
|
||||
// as the node found it now. The peers are replaced whole for the same reason.
|
||||
func (i *Inventory) RecordTunnel(ctx context.Context, nodeID string, t Tunnel) error {
|
||||
if strings.TrimSpace(t.Interface) == "" || strings.TrimSpace(t.PublicKey) == "" {
|
||||
return errors.New("a found tunnel names its interface and its public key, and this names neither")
|
||||
}
|
||||
if _, err := netip.ParsePrefix(t.Range); err != nil {
|
||||
return fmt.Errorf("the found tunnel's range %q is not a range: %w", t.Range, err)
|
||||
}
|
||||
address, err := netip.ParsePrefix(t.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the found tunnel's address %q is not an address with a prefix: %w", t.Address, err)
|
||||
}
|
||||
peers := make([]TunnelPeer, 0, len(t.Peers))
|
||||
for _, p := range t.Peers {
|
||||
host, err := peerHost(p.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the found tunnel's peer %s: %w", shortKey(p.PublicKey), err)
|
||||
}
|
||||
if !address.Masked().Contains(host) {
|
||||
return fmt.Errorf("the found tunnel's peer %s is routed at %s, outside the tunnel's %s",
|
||||
shortKey(p.PublicKey), host, t.Range)
|
||||
}
|
||||
peers = append(peers, TunnelPeer{PublicKey: p.PublicKey, Address: host.String()})
|
||||
}
|
||||
t.Peers = nil
|
||||
t.At = time.Now().UTC()
|
||||
raw, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx, `update node set tunnel = $2 where id = $1`, nodeID, raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `delete from tunnel_peer where node = $1`, nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range peers {
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into tunnel_peer (node, public_key, address) values ($1, $2, $3::inet)`,
|
||||
nodeID, p.PublicKey, p.Address); err != nil {
|
||||
return fmt.Errorf("recording the found tunnel's peer %s: %w", shortKey(p.PublicKey), err)
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// peerHost is the one host address a peer's allowed address names: a bare address, or a /32
|
||||
// (or /128). A wider prefix is refused — a peer routed a whole range is not a machine with an
|
||||
// address the mesh could give a node.
|
||||
func peerHost(allowed string) (netip.Addr, error) {
|
||||
allowed = strings.TrimSpace(allowed)
|
||||
if a, err := netip.ParseAddr(allowed); err == nil {
|
||||
return a, nil
|
||||
}
|
||||
p, err := netip.ParsePrefix(allowed)
|
||||
if err != nil {
|
||||
return netip.Addr{}, fmt.Errorf("%q is not an address", allowed)
|
||||
}
|
||||
if !p.IsSingleIP() {
|
||||
return netip.Addr{}, fmt.Errorf("%q names a range, and a peer of the tunnel is one address", allowed)
|
||||
}
|
||||
return p.Addr(), nil
|
||||
}
|
||||
|
||||
func shortKey(key string) string {
|
||||
if len(key) > 8 {
|
||||
return key[:8] + "…"
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// TunnelOf is the tunnel a node presented, with its peers, or ErrNoTunnel.
|
||||
func (i *Inventory) TunnelOf(ctx context.Context, name string) (Tunnel, error) {
|
||||
var raw []byte
|
||||
var id string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, tunnel from node where name = $1`, name).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return Tunnel{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoTunnel, name)
|
||||
}
|
||||
var t Tunnel
|
||||
if err := json.Unmarshal(raw, &t); err != nil {
|
||||
return Tunnel{}, err
|
||||
}
|
||||
t.Peers, err = i.tunnelPeers(ctx, id)
|
||||
return t, err
|
||||
}
|
||||
|
||||
func (i *Inventory) tunnelPeers(ctx context.Context, nodeID string) ([]TunnelPeer, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select public_key, host(address) from tunnel_peer where node = $1 order by address`, nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []TunnelPeer
|
||||
for rows.Next() {
|
||||
var p TunnelPeer
|
||||
if err := rows.Scan(&p.PublicKey, &p.Address); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// AdoptedTunnel is the tunnel the mesh's private network runs over, if the hub adopted one: the
|
||||
// hub's found tunnel, when the hub is adopted and its overlay key is the tunnel's. Absent, the mesh
|
||||
// runs on its own range — and a hub that found a tunnel but holds another key did not adopt it,
|
||||
// which `overlay show` says.
|
||||
//
|
||||
// The condition on the key is the condition of the whole record: a hub raised with a key of its
|
||||
// own would drop every peer's packets on the found port (novox/hq ADR 0105, option 2), so the
|
||||
// tunnel is adopted only when the hub answers to the key its peers know.
|
||||
func (i *Inventory) AdoptedTunnel(ctx context.Context) (Tunnel, string, bool, error) {
|
||||
var name string
|
||||
var key *string
|
||||
var adopted bool
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select name, overlay_key, adopted from node where is_hub and tunnel is not null`).
|
||||
Scan(&name, &key, &adopted)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Tunnel{}, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Tunnel{}, "", false, err
|
||||
}
|
||||
t, err := i.TunnelOf(ctx, name)
|
||||
if err != nil {
|
||||
return Tunnel{}, "", false, err
|
||||
}
|
||||
if !adopted || key == nil || *key != t.PublicKey {
|
||||
return t, name, false, nil
|
||||
}
|
||||
return t, name, true, nil
|
||||
}
|
||||
|
||||
// CarriedPeers is every peer of the adopted tunnel, with the node that enrolled under its key
|
||||
// where one has: peers of the tunnel, and nodes of the mesh once they enrol. Empty when the hub
|
||||
// adopted no tunnel.
|
||||
func (i *Inventory) CarriedPeers(ctx context.Context) ([]CarriedPeer, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select p.public_key, host(p.address), coalesce(n.name, '')
|
||||
from tunnel_peer p
|
||||
join node hub on hub.id = p.node and hub.is_hub and hub.adopted
|
||||
and hub.tunnel is not null and hub.overlay_key = hub.tunnel->>'public_key'
|
||||
left join node n on n.overlay_key = p.public_key
|
||||
order by p.address`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []CarriedPeer
|
||||
for rows.Next() {
|
||||
var p CarriedPeer
|
||||
if err := rows.Scan(&p.PublicKey, &p.Address, &p.EnrolledAs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Tunnels is every adopted node's found tunnel by node name, for the ones whose overlay key is the
|
||||
// tunnel's — the ones whose private network takes it over. A found tunnel under another key is
|
||||
// left running beside the mesh's, and ADR 0100's rule that the ranges differ applies to it.
|
||||
func (i *Inventory) Tunnels(ctx context.Context) (map[string]Tunnel, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, tunnel from node
|
||||
where adopted and tunnel is not null and overlay_key = tunnel->>'public_key'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]Tunnel{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var raw []byte
|
||||
if err := rows.Scan(&name, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var t Tunnel
|
||||
if err := json.Unmarshal(raw, &t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = t
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RecordCarriedTunnel keeps what a node last said about carrying its found tunnel.
|
||||
func (i *Inventory) RecordCarriedTunnel(ctx context.Context, nodeID string, c Carried) error {
|
||||
c.At = time.Now().UTC()
|
||||
raw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx, `update node set tunnel_carried = $2 where id = $1`, nodeID, raw)
|
||||
return err
|
||||
}
|
||||
|
||||
// CarriedTunnelOf is a node's last account of carrying its found tunnel, and whether it ever gave one.
|
||||
func (i *Inventory) CarriedTunnelOf(ctx context.Context, name string) (Carried, bool, error) {
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx, `select tunnel_carried from node where name = $1`, name).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Carried{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return Carried{}, false, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Carried{}, false, nil
|
||||
}
|
||||
var c Carried
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return Carried{}, false, err
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0105: the mesh adopts the predecessor's tunnel in place. The controller reads the
|
||||
// hub's address and range from the adopted tunnel, assigns an enrolling node the address its key
|
||||
// already had, and refuses to hand out an address the tunnel already holds.
|
||||
|
||||
const (
|
||||
tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key="
|
||||
peerTwo = "PEER-KEY-two============================="
|
||||
peerThree = "PEER-KEY-three==========================="
|
||||
)
|
||||
|
||||
// theFoundTunnel is what a hub presents at enrolment: the predecessor's interface on a
|
||||
// documentation range, with two peers each routed one address.
|
||||
func theFoundTunnel() Tunnel {
|
||||
return Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf",
|
||||
Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey,
|
||||
Peers: []TunnelPeer{{PublicKey: peerTwo, Address: "192.0.2.2/32"},
|
||||
{PublicKey: peerThree, Address: "192.0.2.3"}},
|
||||
}
|
||||
}
|
||||
|
||||
// anAdoptedHub is an adopted node that enrolled with the found tunnel's key and presented the
|
||||
// tunnel, then was placed as the hub — the order genesis does it in.
|
||||
func anAdoptedHub(t *testing.T, inv *Inventory) Node {
|
||||
t.Helper()
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), hub.ID, tunnelKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return hub
|
||||
}
|
||||
|
||||
func TestTheHubsAddressAndRangeComeFromTheAdoptedTunnel(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
hub := anAdoptedHub(t, inv)
|
||||
|
||||
tunnel, name, adopted, err := inv.AdoptedTunnel(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adopted || name != "anchor" || tunnel.Range != "192.0.2.0/24" || tunnel.Port != 51900 {
|
||||
t.Fatalf("the adopted tunnel did not read back: adopted=%t on %s, %+v", adopted, name, tunnel)
|
||||
}
|
||||
if len(tunnel.Peers) != 2 || tunnel.Peers[0].Address != "192.0.2.2" || tunnel.Peers[1].Address != "192.0.2.3" {
|
||||
t.Fatalf("the peers did not read back as one host address each: %+v", tunnel.Peers)
|
||||
}
|
||||
|
||||
// Whatever range the caller would allocate from, the hub is at the tunnel's own address.
|
||||
address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was given %s, not the address the tunnel it took over had", address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEnrollingNodeKeepsTheAddressTheTunnelHadForItsKey(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
anAdoptedHub(t, inv)
|
||||
|
||||
// A predecessor machine enrols: its host took its found interface's key as its overlay key,
|
||||
// which is the key the hub's tunnel already routes to.
|
||||
peer, err := inv.AddNodeAs(t.Context(), "home-server", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), peer.ID, peerThree); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address, err := inv.AssignAddress(t.Context(), peer.ID, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.3" {
|
||||
t.Fatalf("the enrolling peer was given %s, not the 192.0.2.3 the tunnel had for its key", address)
|
||||
}
|
||||
|
||||
carried, err := inv.CarriedPeers(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byKey := map[string]CarriedPeer{}
|
||||
for _, c := range carried {
|
||||
byKey[c.PublicKey] = c
|
||||
}
|
||||
if byKey[peerThree].EnrolledAs != "home-server" || byKey[peerTwo].EnrolledAs != "" {
|
||||
t.Fatalf("the registry cannot say which peer is a node now: %+v", carried)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFreshNodeIsNeverGivenAnAddressTheTunnelHolds(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
anAdoptedHub(t, inv)
|
||||
|
||||
// .1 is the hub, .2 and .3 are peers of the tunnel that have not enrolled: a new machine with
|
||||
// a key of its own gets the next one, from the same range.
|
||||
fresh, err := inv.AddNode(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), fresh.ID, "A-KEY-OF-ITS-OWN========================"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address, err := inv.AssignAddress(t.Context(), fresh.ID, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.4" {
|
||||
t.Fatalf("a fresh node was given %s; 192.0.2.2 and .3 are the tunnel's peers and .1 its hub", address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATunnelUnderAnotherKeyIsNotAdopted(t *testing.T) {
|
||||
// A hub whose overlay key is not the found tunnel's would drop every peer's packets on the
|
||||
// found port (ADR 0105, option 2). Such a tunnel is recorded and not adopted: the mesh keeps
|
||||
// its own range, and ADR 0100's non-overlap rule stands for it.
|
||||
inv := fresh(t)
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), hub.ID, "THE-MESHS-OWN-KEY======================="); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51820", "hosting", true, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, adopted, err := inv.AdoptedTunnel(t.Context()); err != nil || adopted {
|
||||
t.Fatalf("a tunnel under another key was adopted (err %v)", err)
|
||||
}
|
||||
if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 0 {
|
||||
t.Fatalf("peers of a tunnel that was not adopted are carried: %+v (err %v)", carried, err)
|
||||
}
|
||||
if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" {
|
||||
t.Fatalf("the hub was given %s (err %v); it should allocate from the mesh's own range", address, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPeerRoutedARangeIsRefused(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := theFoundTunnel()
|
||||
found.Peers = append(found.Peers, TunnelPeer{PublicKey: "WIDE", Address: "198.51.100.0/24"})
|
||||
err = inv.RecordTunnel(t.Context(), hub.ID, found)
|
||||
if err == nil || !strings.Contains(err.Error(), "names a range") {
|
||||
t.Fatalf("a peer routed a whole range was recorded as a machine with an address: %v", err)
|
||||
}
|
||||
if _, err := inv.TunnelOf(t.Context(), "anchor"); !errors.Is(err, ErrNoTunnel) {
|
||||
t.Fatalf("a refused tunnel was recorded anyway: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -124,6 +124,29 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
||||
"%s's overlay key could not be recorded: %w", node.Name, err)
|
||||
}
|
||||
}
|
||||
// And the tunnel it found, whose key is the overlay key above (novox/hq ADR 0105). Recorded
|
||||
// before the token is spent for the same reason as the keys: the first declaration this node
|
||||
// receives is composed from it, and a hub enrolled without its tunnel would be placed at an
|
||||
// address of the mesh's choosing rather than the tunnel's.
|
||||
if request.Tunnel != nil {
|
||||
if request.Tunnel.PublicKey != request.OverlayKey {
|
||||
return EnrolReply{}, fmt.Errorf("%s presented a tunnel under key %s and an overlay key "+
|
||||
"that is not it; a tunnel is taken over with its own key or not at all", node.Name,
|
||||
request.Tunnel.PublicKey)
|
||||
}
|
||||
peers := make([]inventory.TunnelPeer, 0, len(request.Tunnel.Peers))
|
||||
for _, p := range request.Tunnel.Peers {
|
||||
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
||||
}
|
||||
if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
|
||||
Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
|
||||
Config: request.Tunnel.Config, Port: request.Tunnel.Port,
|
||||
Address: request.Tunnel.Address, Range: request.Tunnel.Range,
|
||||
PublicKey: request.Tunnel.PublicKey, Peers: peers,
|
||||
}); err != nil {
|
||||
return EnrolReply{}, fmt.Errorf("%s's found tunnel could not be recorded: %w", node.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Spent once the node is complete in the store.
|
||||
if err := e.Inventory.Spend(ctx, secret, by); err != nil {
|
||||
@@ -219,6 +242,15 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
|
||||
if report.Tunnel != nil {
|
||||
if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{
|
||||
Interface: report.Tunnel.Interface, Port: report.Tunnel.Port, Range: report.Tunnel.Range,
|
||||
Peers: report.Tunnel.Peers, Taken: report.Tunnel.Taken, Kept: report.Tunnel.Kept,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// A bare word that a node is there is not an account of what the machine did or holds: it
|
||||
// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
|
||||
|
||||
@@ -71,12 +71,39 @@ type EnrolRequest struct {
|
||||
// node's public key could replay the spent token (novox/hq issue 083, on review).
|
||||
Proof []byte `json:"proof,omitempty"`
|
||||
|
||||
// Tunnel is the tunnel this node found on its machine and whose key it took as its overlay
|
||||
// key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented
|
||||
// with the keys because it is one of them — OverlayKey above is this tunnel's public key when
|
||||
// it is set — and the mesh composes the hub's address, the range and every carried peer from
|
||||
// it. Nil from a node that found none, which is every converged one.
|
||||
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
||||
|
||||
// Redelivered is set by the control plane, never sent: the broker handed this request over a
|
||||
// second time. Such a request does not finish an enrolment already spent — the first time may
|
||||
// have answered, and the node holds what it was told.
|
||||
Redelivered bool `json:"-"`
|
||||
}
|
||||
|
||||
// Tunnel is a found tunnel as a node presents it: everything but its private key, which the node
|
||||
// keeps as its own overlay key and never sends.
|
||||
type Tunnel struct {
|
||||
Interface string `json:"interface"`
|
||||
Unit string `json:"unit"`
|
||||
Config string `json:"config"`
|
||||
Port int `json:"port"`
|
||||
Address string `json:"address"`
|
||||
Range string `json:"range"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Peers []TunnelPeer `json:"peers,omitempty"`
|
||||
}
|
||||
|
||||
// TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to
|
||||
// it.
|
||||
type TunnelPeer struct {
|
||||
PublicKey string `json:"public_key"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// Signed is a declaration and the signature over it.
|
||||
//
|
||||
// The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies
|
||||
@@ -129,6 +156,21 @@ type Report struct {
|
||||
// Reachable is what can be reached on the machine now: every listening socket and every
|
||||
// published container port. Only an adopted node reports it; it is what converging previews.
|
||||
Reachable []Reach `json:"reachable,omitempty"`
|
||||
|
||||
// Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR
|
||||
// 0105): the interface, its port, range and peer count, whether the found interface is down
|
||||
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||
}
|
||||
|
||||
// CarriedTunnel is a node's account of the tunnel it took over.
|
||||
type CarriedTunnel struct {
|
||||
Interface string `json:"interface"`
|
||||
Port int `json:"port"`
|
||||
Range string `json:"range"`
|
||||
Peers int `json:"peers"`
|
||||
Taken bool `json:"taken"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// Held is one file or container found on an adopted node and kept as it was.
|
||||
|
||||
@@ -43,6 +43,40 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
|
||||
keyPath = DefaultKeyPath
|
||||
}
|
||||
|
||||
up := Resource{
|
||||
"id": "overlay-up", "type": "service", "unit": Unit,
|
||||
"state": "running",
|
||||
// Enabled, so the node comes back onto the network after a reboot without waiting to
|
||||
// be told again. A node whose overlay only exists while something is watching is not
|
||||
// a node that survives being switched off and on.
|
||||
"boot": "enabled",
|
||||
// And restarted when the peer list changes, because a running interface does not
|
||||
// re-read its configuration.
|
||||
//
|
||||
// This is the whole of it: a node joins, every existing node's peer list changes,
|
||||
// each file is replaced — and without this the service is already running, nothing
|
||||
// reloads it, and every node keeps a network that no longer matches the mesh. It
|
||||
// reports complete success. The lab found it the moment a third node arrived.
|
||||
//
|
||||
// Declared state rather than a command. The service must reflect the file; the host
|
||||
// works out that it does not. A command to restart would be an action, and the link
|
||||
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
|
||||
// which is how this shape was arrived at.
|
||||
"restart-on": []string{"overlay-config"},
|
||||
}
|
||||
if node.TakesOver != nil {
|
||||
// The private network takes over the tunnel it found (novox/hq ADR 0105): before this
|
||||
// unit starts, the host stops and disables the found one — never flushing it — and keeps
|
||||
// its configuration like any held file. The key is already the found one: the node took
|
||||
// it as its own overlay key when it enrolled, which is why the mesh's peer list for it
|
||||
// carries the found peers under the key they know.
|
||||
up["takes-over"] = map[string]any{
|
||||
"interface": node.TakesOver.Interface,
|
||||
"unit": node.TakesOver.Unit,
|
||||
"config": node.TakesOver.Config,
|
||||
}
|
||||
}
|
||||
|
||||
resources := []Resource{
|
||||
{
|
||||
"id": "overlay-tools", "type": "package", "package": "wireguard-tools",
|
||||
@@ -55,27 +89,7 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
|
||||
"mode": "0600",
|
||||
"content": config(node, peers, keyPath),
|
||||
},
|
||||
{
|
||||
"id": "overlay-up", "type": "service", "unit": Unit,
|
||||
"state": "running",
|
||||
// Enabled, so the node comes back onto the network after a reboot without waiting to
|
||||
// be told again. A node whose overlay only exists while something is watching is not
|
||||
// a node that survives being switched off and on.
|
||||
"boot": "enabled",
|
||||
// And restarted when the peer list changes, because a running interface does not
|
||||
// re-read its configuration.
|
||||
//
|
||||
// This is the whole of it: a node joins, every existing node's peer list changes,
|
||||
// each file is replaced — and without this the service is already running, nothing
|
||||
// reloads it, and every node keeps a network that no longer matches the mesh. It
|
||||
// reports complete success. The lab found it the moment a third node arrived.
|
||||
//
|
||||
// Declared state rather than a command. The service must reflect the file; the host
|
||||
// works out that it does not. A command to restart would be an action, and the link
|
||||
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
|
||||
// which is how this shape was arrived at.
|
||||
"restart-on": []string{"overlay-config"},
|
||||
},
|
||||
up,
|
||||
}
|
||||
|
||||
// The names used to be appended here, on the argument that a node with peers and no names is
|
||||
|
||||
@@ -223,3 +223,40 @@ func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) {
|
||||
"compromised one could do")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: a node whose private network takes over the tunnel it found is told so on
|
||||
// the interface's service, and nothing else about the declaration changes — the key is already
|
||||
// the found one, taken at enrolment.
|
||||
func TestTakingOverAFoundTunnelIsSaidOnTheInterfacesService(t *testing.T) {
|
||||
node := Node{Name: "anchor", Key: "PUB", Address: "192.0.2.1", Hub: true,
|
||||
Endpoint: "198.51.100.1:51900",
|
||||
TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf"}}
|
||||
config, resources := declarationFor(t, node, nil)
|
||||
|
||||
var up map[string]any
|
||||
for _, r := range resources {
|
||||
if r["type"] == "service" {
|
||||
up = r
|
||||
}
|
||||
}
|
||||
takes, ok := up["takes-over"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("the interface's service does not say what it takes over: %+v", up)
|
||||
}
|
||||
if takes["unit"] != "wg-quick@wg0" || takes["config"] != "/etc/wireguard/wg0.conf" || takes["interface"] != "wg0" {
|
||||
t.Errorf("the takeover names the wrong tunnel: %+v", takes)
|
||||
}
|
||||
if !strings.Contains(config, "ListenPort = 51900") {
|
||||
t.Errorf("the hub's interface does not listen on the tunnel's port:\n%s", config)
|
||||
}
|
||||
if strings.Contains(config, "PrivateKey") {
|
||||
t.Error("the found key travelled in the configuration; it is the node's own, set from its key file")
|
||||
}
|
||||
|
||||
_, plain := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "192.0.2.4"}, nil)
|
||||
for _, r := range plain {
|
||||
if _, says := r["takes-over"]; says {
|
||||
t.Error("a node taking over nothing was told to take something over")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,40 @@ type Node struct {
|
||||
Site string
|
||||
Hub bool
|
||||
Address string
|
||||
|
||||
// Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the
|
||||
// mesh has no record of, each known by the public key and the address the found tunnel routed
|
||||
// to it. Only a hub has any. They stay in its peer list until a node enrols with that key —
|
||||
// from then on the node is the peer.
|
||||
Carried []Carried
|
||||
// TakesOver names the found tunnel this node's private network replaces: its unit is stopped
|
||||
// and disabled, never flushed, and its configuration kept, before the mesh's interface comes
|
||||
// up with the found key. Nil on a node that raises the mesh's interface beside whatever it has.
|
||||
TakesOver *TakeOver
|
||||
}
|
||||
|
||||
// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a
|
||||
// node of the mesh.
|
||||
type Carried struct {
|
||||
Key string
|
||||
Address string
|
||||
}
|
||||
|
||||
// TakeOver is the found tunnel a node's private network takes over, as the host is told it.
|
||||
type TakeOver struct {
|
||||
Interface string
|
||||
Unit string
|
||||
Config string
|
||||
}
|
||||
|
||||
// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named
|
||||
// by the key its packets arrive under.
|
||||
func CarriedName(key string) string {
|
||||
short := key
|
||||
if len(short) > 8 {
|
||||
short = short[:8] + "…"
|
||||
}
|
||||
return "a peer of the tunnel (" + short + ")"
|
||||
}
|
||||
|
||||
// Reachable reports whether other nodes can dial this one. Declared, never inferred.
|
||||
@@ -145,7 +179,9 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
|
||||
// The hub holds every node that does not share a site with it, because those nodes
|
||||
// route through it and it must know where to send the replies. Ones it cannot dial
|
||||
// will dial it.
|
||||
enrolled := map[string]bool{}
|
||||
for _, other := range usable {
|
||||
enrolled[other.Key] = true
|
||||
if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
|
||||
continue
|
||||
}
|
||||
@@ -156,6 +192,21 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
|
||||
Why: "routes through this hub",
|
||||
})
|
||||
}
|
||||
// And every peer of the tunnel it took over that has not enrolled (novox/hq ADR
|
||||
// 0105): the same key and the same address the found tunnel had for it, so the
|
||||
// machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in,
|
||||
// as it always did. Once a node enrols with that key, the node's entry above is the
|
||||
// peer, and WireGuard takes one entry per key.
|
||||
for _, c := range self.Carried {
|
||||
if enrolled[c.Key] {
|
||||
continue
|
||||
}
|
||||
peers = append(peers, Peer{
|
||||
Name: CarriedName(c.Key), Key: c.Key,
|
||||
Allowed: c.Address + "/32",
|
||||
Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
|
||||
|
||||
@@ -304,3 +304,39 @@ func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) {
|
||||
"nowhere to go")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: a hub that took over the predecessor's tunnel carries every peer that tunnel
|
||||
// had, under the key and at the address the peer knows, until a node enrols with that key.
|
||||
func TestTheHubCarriesTheTunnelsPeersUntilTheyEnrol(t *testing.T) {
|
||||
hub := at("anchor", "hosting", "192.0.2.1", "198.51.100.1:51900", true)
|
||||
hub.Carried = []Carried{
|
||||
{Key: "key-home", Address: "192.0.2.2"},
|
||||
{Key: "key-workstation", Address: "192.0.2.3"},
|
||||
}
|
||||
// The machine behind key-home enrolled: it is a node now, at the address it kept.
|
||||
home := at("home", "house", "192.0.2.2", "", false)
|
||||
home.Key = "key-home"
|
||||
|
||||
g, err := Compute([]Node{hub, home}, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peers := peersOf(t, g, "anchor")
|
||||
carried, ok := peers[CarriedName("key-workstation")]
|
||||
if !ok {
|
||||
t.Fatalf("the hub does not carry the peer that has not enrolled: %+v", g["anchor"])
|
||||
}
|
||||
if carried.Allowed != "192.0.2.3/32" || carried.Endpoint != "" || carried.Key != "key-workstation" {
|
||||
t.Errorf("a carried peer is not the tunnel's own entry — same key, its one address, no endpoint: %+v", carried)
|
||||
}
|
||||
if _, twice := peers[CarriedName("key-home")]; twice {
|
||||
t.Error("a peer that enrolled is carried as well as listed as a node: WireGuard takes one entry per key")
|
||||
}
|
||||
if node, ok := peers["home"]; !ok || node.Key != "key-home" || node.Allowed != "192.0.2.2/32" {
|
||||
t.Errorf("the enrolled peer is not the node it became: %+v", node)
|
||||
}
|
||||
// Carried peers are the hub's business only: a spoke routes everything through the hub.
|
||||
if _, leaked := peersOf(t, g, "home")[CarriedName("key-workstation")]; leaked {
|
||||
t.Error("a carried peer appeared in a spoke's peer list")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Lab bed: the hub adopts the predecessor's tunnel (novox/hq ADR 0105)
|
||||
|
||||
A scenario and an integration-test skeleton for the mesh-lab repository, kept here because this
|
||||
branch changes only the controller and the host. Move `adopt-the-tunnel.yml` to
|
||||
`mesh-lab/scenarios/` and `adopt-the-tunnel.test.ts` to `mesh-lab/test/integration/` when the
|
||||
feature lands; neither has been run. The skeleton follows `adoption.test.ts` and reuses its
|
||||
harness. Documentation addresses throughout; the bed is node-agnostic.
|
||||
|
||||
## The bed, precisely
|
||||
|
||||
Three machines on one public segment, `hosting` (192.0.2.0/24), inbound allowed on all (the
|
||||
anchor's firewall is the predecessor's, installed by the bed):
|
||||
|
||||
| machine | address | role |
|
||||
|---|---|---|
|
||||
| `anchor` | 192.0.2.10 | the machine in use: the predecessor's hub, then the mesh adopted on it |
|
||||
| `peer-a` | 192.0.2.20 | a predecessor machine: reaches a service on the anchor through the tunnel; later **enrols and keeps its address** |
|
||||
| `peer-b` | 192.0.2.30 | a second predecessor machine: reaches the same service; **never enrols** — the peer that must notice nothing throughout |
|
||||
| `fresh` | 192.0.2.40 | a new machine: enrols later and **gets a fresh address from the same range** |
|
||||
|
||||
**Prepared the way the predecessor leaves a hub** (before genesis, by the bed, on `anchor`):
|
||||
|
||||
- `wireguard-tools` installed; a keypair made on each of `anchor`, `peer-a`, `peer-b`.
|
||||
- `/etc/wireguard/wg0.conf` on the anchor: `[Interface]` `PrivateKey = <anchor's>`,
|
||||
`ListenPort = 51900`, `Address = 10.10.0.1/24`; two `[Peer]` sections — `peer-a`'s public
|
||||
key with `AllowedIPs = 10.10.0.2/32`, `peer-b`'s with `AllowedIPs = 10.10.0.3/32`. Raised with
|
||||
`systemctl enable --now wg-quick@wg0`. **10.10.0.0/24 is deliberately not the mesh's default
|
||||
range** (10.42.0.0/16), so a hub address in 10.10.0.0/24 can only have come from the tunnel.
|
||||
- `wg0.conf` on each peer: its own key, `Address = 10.10.0.2/24` (resp. `.3/24`), one
|
||||
`[Peer]` — the anchor's public key, `Endpoint = 192.0.2.10:51900`,
|
||||
`AllowedIPs = 10.10.0.0/24`, `PersistentKeepalive = 25`. Raised the same way.
|
||||
- A service on the anchor the peers reach **only over the tunnel**: a container publishing
|
||||
`10.10.0.1:8081:80` (bound to the tunnel address, so a call from 192.0.2.20 to 10.10.0.1:8081
|
||||
proves the tunnel carried it). Under a name no catalogue module uses — this bed is about the
|
||||
tunnel, not about taking a service.
|
||||
- The predecessor's firewall (`ufw`) allowing `51900/udp` and `22/tcp`, denying the rest — as ADR
|
||||
0100's bed prepares it.
|
||||
- A record of the anchor's `wg0` public key and of `sha256sum /etc/wireguard/wg0.conf`, taken
|
||||
before genesis, for the assertions below.
|
||||
|
||||
**Genesis**, adopted, on the anchor: `mesh-bootstrap --adopted --node anchor --site hosting
|
||||
--endpoint 192.0.2.10:51900 …` — no `--hub-port`, no `--overlay-range`, no `--tunnel`: the
|
||||
installer finds `wg0` itself (one interface besides `mesh0`) and takes its port and range. The
|
||||
bed asserts genesis **says** it found and took the tunnel.
|
||||
|
||||
## Assertions, in the record's order
|
||||
|
||||
- **T1 — the tunnel changes hands and the peers notice nothing.** After genesis and the push
|
||||
that raises the private network on the anchor:
|
||||
- `wg show interfaces` on the anchor lists `mesh0` and not `wg0`;
|
||||
`systemctl is-active wg-quick@wg0` is inactive and `is-enabled` disabled;
|
||||
- `/etc/wireguard/wg0.conf` is on disk with the recorded digest (kept, never flushed), and
|
||||
`node show anchor` names where its original was kept;
|
||||
- `wg show mesh0 public-key` is the anchor's recorded `wg0` public key; `wg show mesh0
|
||||
listen-port` is 51900; `ip -o addr show dev mesh0` carries `10.10.0.1`; `wg show mesh0 peers`
|
||||
lists both peers' public keys with their `/32` allowed addresses;
|
||||
- a loop on `peer-a` and `peer-b` calling `http://10.10.0.1:8081/` every second, started before
|
||||
genesis, records **no window of failure longer than one WireGuard re-handshake** (measure and
|
||||
assert an upper bound — the switch is one unit stop plus one unit start on the anchor); the
|
||||
peers' `wg0.conf` digests are unchanged; the peers' `wg show wg0 latest-handshakes` advance
|
||||
after the switch.
|
||||
- `overlay show` lists `anchor` as the hub over the tunnel it took over, and both peers under
|
||||
"peers of the tunnel … not nodes of the mesh", not yet enrolled.
|
||||
- **T2 — a peer enrols and keeps its address.** On `peer-a`: `node add peer-a --adopted`,
|
||||
token issued, `mesh-host enrol --token …` **with the broker reached over the tunnel** (the
|
||||
broker address in the token is `10.10.0.1:<bus>`, which only the tunnel routes); then `overlay
|
||||
place peer-a --site house` and a push. Assert: `node show peer-a` says a tunnel `wg0` was
|
||||
found and `overlay show` puts `peer-a` at **10.10.0.2**; the carried-peers list now says
|
||||
`enrolled as peer-a`; the anchor's `mesh0` still has exactly one entry for `peer-a`'s key;
|
||||
`peer-a`'s `wg0` is down and `mesh0` up with the same key; `peer-a` still reaches
|
||||
`10.10.0.1:8081` and `peer-b` still does too, uninterrupted.
|
||||
- **T3 — a new machine gets a fresh address from the same range.** `fresh` enrols converged
|
||||
(no tunnel), is placed, pushed. Assert its address is **10.10.0.4** (`.1` hub, `.2` and `.3`
|
||||
the tunnel's), that it reaches `10.10.0.1` (the hub) and `10.10.0.2` (the enrolled peer) —
|
||||
`ping -c1` over `mesh0` — and that `peer-b`, never enrolled, is still served.
|
||||
- **T4 — nothing derived from the address is stale.** `plan anchor --json` and `plan peer-a
|
||||
--json` (and the rendered `/etc/hosts` on each node) name `10.10.0.1` for the anchor and
|
||||
`10.10.0.2` for `peer-a`, and no address in `10.42.0.0/16`; the broker address handed to a
|
||||
module issued on `peer-a` is `anchor.internal:<bus>` resolving to `10.10.0.1`; the same after a
|
||||
second `push` of every node, byte for byte.
|
||||
- **N — the narrowed ADR 0100 check.** On `fresh`, a converged genesis dry-run with
|
||||
`--overlay-range 10.10.0.0/24` while a *second* tunnel the bed raises there (`wg1` at
|
||||
10.10.0.9/24, not adopted because the node is converged) is up, is refused naming `wg1` —
|
||||
the non-overlap rule still applies where a tunnel is not adopted.
|
||||
|
||||
## What is not asserted here
|
||||
|
||||
- Taking a service over the tunnel (ADR 0100's bed does that).
|
||||
- IPv6 tunnels: the parser reads them, the bed prepares only IPv4.
|
||||
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). Skeleton — NOT YET RUN.
|
||||
*
|
||||
* The bed and every assertion are described in README.md beside this file; the numbered
|
||||
* assertions here are that document's. Follows adoption.test.ts: same harness, same `on`/`must`
|
||||
* helpers, same genesis wrapper.
|
||||
*
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, catalogueIsPresent } from "./harness.ts";
|
||||
import { genesis } from "./genesis.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const skip = !capability.usable ? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
|
||||
: catalogueIsPresent();
|
||||
|
||||
const SCENARIO = "adopt-the-tunnel";
|
||||
const ANCHOR = "anchor", PEER_A = "peer-a", PEER_B = "peer-b", FRESH = "fresh";
|
||||
const TUNNEL = { port: 51900, range: "10.10.0.0/24", hub: "10.10.0.1", a: "10.10.0.2", b: "10.10.0.3", fresh: "10.10.0.4" };
|
||||
const SERVICE = `http://${TUNNEL.hub}:8081/`;
|
||||
|
||||
let instanceId = "";
|
||||
let wg0Key = ""; // the anchor's wg0 public key, recorded before genesis
|
||||
let wg0Digest = ""; // sha256 of /etc/wireguard/wg0.conf before genesis
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
/** The controller, a container on the anchor. */
|
||||
async function control(args: string): Promise<string> {
|
||||
return must(ANCHOR, `docker exec mesh-controller mesh-controller ${args}`);
|
||||
}
|
||||
|
||||
/** Prepares a machine the way the predecessor leaves it: a keypair and a wg0 — see README.md. */
|
||||
async function predecessorTunnelOn(machine: string, conf: (keys: Record<string, string>) => string, keys: Record<string, string>): Promise<void> {
|
||||
await must(machine, "apt-get install -y wireguard-tools >/dev/null 2>&1 || pacman -S --noconfirm wireguard-tools >/dev/null");
|
||||
await must(machine, `umask 077; printf '%s' '${conf(keys)}' > /etc/wireguard/wg0.conf`);
|
||||
await must(machine, "systemctl enable --now wg-quick@wg0");
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
await destroyAll(SCENARIO);
|
||||
const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
|
||||
instanceId = (await raise(scenario)).instanceId;
|
||||
|
||||
// Keys for the three predecessor machines, made where they live and never moved.
|
||||
const keys: Record<string, string> = {};
|
||||
for (const m of [ANCHOR, PEER_A, PEER_B]) {
|
||||
await must(m, "umask 077; wg genkey > /etc/wireguard/predecessor.key");
|
||||
keys[m] = (await must(m, "wg pubkey < /etc/wireguard/predecessor.key")).trim();
|
||||
}
|
||||
await predecessorTunnelOn(ANCHOR, k => [
|
||||
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `ListenPort = ${TUNNEL.port}`, `Address = ${TUNNEL.hub}/24`,
|
||||
"[Peer]", `PublicKey = ${k[PEER_A]}`, `AllowedIPs = ${TUNNEL.a}/32`,
|
||||
"[Peer]", `PublicKey = ${k[PEER_B]}`, `AllowedIPs = ${TUNNEL.b}/32`,
|
||||
].join("\n"), keys);
|
||||
for (const [m, addr] of [[PEER_A, TUNNEL.a], [PEER_B, TUNNEL.b]] as const) {
|
||||
await predecessorTunnelOn(m, k => [
|
||||
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `Address = ${addr}/24`,
|
||||
"[Peer]", `PublicKey = ${k[ANCHOR]}`, `Endpoint = 192.0.2.10:${TUNNEL.port}`, `AllowedIPs = ${TUNNEL.range}`, "PersistentKeepalive = 25",
|
||||
].join("\n"), keys);
|
||||
}
|
||||
// A service reachable only over the tunnel, under a name no catalogue module uses.
|
||||
await must(ANCHOR, `docker run -d --name predecessor-page -p ${TUNNEL.hub}:8081:80 nginx:alpine`);
|
||||
// The predecessor's firewall: the tunnel's port and ssh, nothing else (as ADR 0100's bed).
|
||||
await must(ANCHOR, `ufw --force reset >/dev/null; ufw default deny incoming; ufw allow 22/tcp; ufw allow ${TUNNEL.port}/udp; ufw --force enable`);
|
||||
for (const m of [PEER_A, PEER_B]) assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service over the tunnel before genesis`);
|
||||
|
||||
wg0Key = (await must(ANCHOR, "wg show wg0 public-key")).trim();
|
||||
wg0Digest = (await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0];
|
||||
// The probe the peers keep running through the switch: one call a second, failures counted.
|
||||
for (const m of [PEER_A, PEER_B]) await must(m, `nohup sh -c 'while :; do curl -fsS --max-time 1 ${SERVICE} >/dev/null 2>&1 || date +%s >> /tmp/failed; sleep 1; done' >/dev/null 2>&1 &`);
|
||||
});
|
||||
|
||||
after(async () => { if (instanceId) await destroy(instanceId); });
|
||||
|
||||
test("T1 — adopted, the tunnel changes hands and the peers notice nothing", { skip }, async () => {
|
||||
const ran = await genesis({ instanceId, machine: ANCHOR, node: ANCHOR, site: "hosting",
|
||||
flags: ["--adopted", "--endpoint", `192.0.2.10:${TUNNEL.port}`] } as never);
|
||||
assert.match(ran.said, /tunnel\s+wg0/i, `genesis did not say it found and took the tunnel:\n${ran.said}`);
|
||||
|
||||
const ifaces = await must(ANCHOR, "wg show interfaces");
|
||||
assert.match(ifaces, /\bmesh0\b/); assert.doesNotMatch(ifaces, /\bwg0\b/);
|
||||
assert.equal((await on(ANCHOR, "systemctl is-active wg-quick@wg0")).out.trim(), "inactive");
|
||||
assert.equal((await on(ANCHOR, "systemctl is-enabled wg-quick@wg0")).out.trim(), "disabled");
|
||||
assert.equal((await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0], wg0Digest, "the found configuration was changed or flushed");
|
||||
assert.equal((await must(ANCHOR, "wg show mesh0 public-key")).trim(), wg0Key, "the mesh's interface is not up with the found key");
|
||||
assert.equal((await must(ANCHOR, "wg show mesh0 listen-port")).trim(), String(TUNNEL.port));
|
||||
assert.match(await must(ANCHOR, "ip -o addr show dev mesh0"), new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
|
||||
const peers = await must(ANCHOR, "wg show mesh0 allowed-ips");
|
||||
assert.match(peers, new RegExp(`${TUNNEL.a}/32`)); assert.match(peers, new RegExp(`${TUNNEL.b}/32`));
|
||||
|
||||
for (const m of [PEER_A, PEER_B]) {
|
||||
const failed = (await on(m, "cat /tmp/failed 2>/dev/null | wc -l")).out.trim();
|
||||
assert.ok(Number(failed) <= 5, `${m} lost the service for ${failed} seconds through the switch`);
|
||||
assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service after the switch`);
|
||||
}
|
||||
const shown = await control("overlay show");
|
||||
assert.match(shown, /anchor.*hub.*took over on wg0/);
|
||||
assert.match(shown, /peers of the tunnel/); assert.match(shown, /not yet enrolled/);
|
||||
assert.match(await control(`node show ${ANCHOR}`), /tunnel found\s+wg0 on port 51900/);
|
||||
});
|
||||
|
||||
test("T2 — a peer enrols over the tunnel and keeps its address", { skip }, async () => {
|
||||
await control(`node add ${PEER_A} --adopted`);
|
||||
const token = (await control(`token issue --node ${PEER_A}`)).match(/token\s+(\S+)/)?.[1] ?? "";
|
||||
// The token's broker address is the hub's tunnel address: only the tunnel routes it.
|
||||
await must(PEER_A, `mesh-host enrol --token '${token}'`);
|
||||
await control(`overlay place ${PEER_A} --site house`);
|
||||
await control(`assign ${PEER_A} networking`);
|
||||
await control(`push ${PEER_A} --wait 2m`);
|
||||
|
||||
assert.match(await control("overlay show"), new RegExp(`${PEER_A}\\s+${TUNNEL.a.replaceAll(".", "\\.")}`));
|
||||
assert.match(await control("overlay show"), new RegExp(`enrolled as ${PEER_A}`));
|
||||
const onHub = await must(ANCHOR, "wg show mesh0 allowed-ips");
|
||||
assert.equal(onHub.split("\n").filter(l => l.includes(`${TUNNEL.a}/32`)).length, 1, "the enrolled peer's key appears twice on the hub");
|
||||
assert.doesNotMatch(await must(PEER_A, "wg show interfaces"), /\bwg0\b/);
|
||||
assert.ok((await on(PEER_A, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
|
||||
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok, "the peer that never enrols lost the service");
|
||||
});
|
||||
|
||||
test("T3 — a new machine gets a fresh address from the same range", { skip }, async () => {
|
||||
await control(`node add ${FRESH}`);
|
||||
const token = (await control(`token issue --node ${FRESH}`)).match(/token\s+(\S+)/)?.[1] ?? "";
|
||||
await must(FRESH, `mesh-host enrol --token '${token}'`);
|
||||
await control(`overlay place ${FRESH} --nothing`);
|
||||
await control(`assign ${FRESH} networking`);
|
||||
await control(`push ${FRESH} --wait 2m`);
|
||||
assert.match(await control("overlay show"), new RegExp(`${FRESH}\\s+${TUNNEL.fresh.replaceAll(".", "\\.")}`));
|
||||
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.hub}`)).ok, "the new machine does not reach the hub");
|
||||
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.a}`)).ok, "the new machine does not reach the enrolled peer");
|
||||
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
|
||||
});
|
||||
|
||||
test("T4 — nothing derived from the address is stale", { skip }, async () => {
|
||||
for (const n of [ANCHOR, PEER_A, FRESH]) {
|
||||
const plan = await control(`plan ${n} --json`);
|
||||
assert.doesNotMatch(plan, /10\.42\./, `${n}'s plan names the mesh's default range`);
|
||||
assert.match(plan, new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
|
||||
assert.match(await must(n, "cat /etc/hosts"), new RegExp(`${TUNNEL.hub.replaceAll(".", "\\.")}\\s+anchor\\.internal`));
|
||||
}
|
||||
const first = await control(`plan ${PEER_A} --json`);
|
||||
await control("push");
|
||||
assert.equal(await control(`plan ${PEER_A} --json`), first, "a push changed what the plan says");
|
||||
});
|
||||
|
||||
test("N — where a tunnel is not adopted, the ranges must still differ (ADR 0100)", { skip }, async () => {
|
||||
await must(FRESH, "umask 077; printf '[Interface]\\nPrivateKey = %s\\nAddress = 10.10.0.9/24\\n' \"$(wg genkey)\" > /etc/wireguard/wg1.conf; systemctl start wg-quick@wg1");
|
||||
const ran = await genesis({ instanceId, machine: FRESH, node: FRESH, flags: ["--dry-run", "--overlay-range", TUNNEL.range], attempts: 1, verify: false, hostService: false } as never);
|
||||
assert.match(ran.said, /wg1/, `a converged genesis did not refuse the overlapping tunnel it does not adopt:\n${ran.said}`);
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
# THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). See README.md beside this file.
|
||||
#
|
||||
# hosting (public)
|
||||
# anchor 192.0.2.10 the predecessor's hub: wg0 on udp/51900, 10.10.0.1/24, two peers; then the
|
||||
# mesh adopted on it, taking the tunnel over
|
||||
# peer-a 192.0.2.20 a predecessor machine reaching a service on the anchor over the tunnel;
|
||||
# enrols later and keeps 10.10.0.2
|
||||
# peer-b 192.0.2.30 a predecessor machine that never enrols: must notice nothing, ever
|
||||
# fresh 192.0.2.40 a new machine: enrols later and gets 10.10.0.4
|
||||
#
|
||||
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the bed.
|
||||
scenario: adopt-the-tunnel
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 12GiB
|
||||
cpus: 6
|
||||
disk: 60GiB
|
||||
peer-a:
|
||||
at: { segment: hosting, address: [192.0.2.20] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
disk: 20GiB
|
||||
peer-b:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
disk: 15GiB
|
||||
fresh:
|
||||
at: { segment: hosting, address: [192.0.2.40] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
disk: 20GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
Reference in New Issue
Block a user