Adopt the predecessor's tunnel in place: its range, its address, its peers (hq ADR 0105) #49

Merged
jschoubben merged 2 commits from feat/adopt-the-tunnel into main 2026-09-23 22:38:31 +00:00
16 changed files with 1351 additions and 42 deletions
Showing only changes of commit 3c836f0abb - Show all commits
+34
View File
@@ -45,6 +45,9 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
return nil
}
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
if err := showTunnel(ctx, inv, node.Name); err != nil {
return err
}
if len(said.Held) == 0 {
fmt.Printf(" holding nothing found\n")
}
@@ -63,6 +66,37 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
return nil
}
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
tunnel, err := inv.TunnelOf(ctx, name)
if errors.Is(err, inventory.ErrNoTunnel) {
return nil
}
if err != nil {
return err
}
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
carried, said, err := inv.CarriedTunnelOf(ctx, name)
if err != nil {
return err
}
switch {
case !said:
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
case carried.Taken:
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
default:
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
}
if said && carried.Kept != "" {
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
}
return nil
}
func orNone(s string) string {
if s == "" {
return "none reported"
+121 -9
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
@@ -21,11 +22,28 @@ import (
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.
func overlayCIDR() string {
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
return v
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
const DefaultOverlayCIDR = "10.42.0.0/16"
// overlayRange is the range the mesh allocates node addresses from.
//
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
// found is at that tunnel's address, its peers are at theirs, and every node's address is
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
// the range genesis was told, or the default.
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
if err != nil {
return "", err
}
return "10.42.0.0/16"
if adopted {
return tunnel.Range, nil
}
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
return v, nil
}
return DefaultOverlayCIDR, nil
}
func overlayCommand(ctx context.Context, args []string) error {
@@ -110,16 +128,46 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
}
}
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
// have the mesh's interface up where no peer is listening for it.
found, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
var tunnel inventory.Tunnel
adoptsTunnel := false
if *hub && found.Adopted {
if t, err := inv.TunnelOf(ctx, node); err == nil {
tunnel = t
placed, _ := inv.Overlays(ctx)
for _, o := range placed {
if o.Name == node && o.Key == t.PublicKey {
adoptsTunnel = true
}
}
} else if !errors.Is(err, inventory.ErrNoTunnel) {
return err
}
}
if adoptsTunnel {
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
}
}
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
// election by address prefix fails silently (novox/hq ADR 0007).
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
return err
}
found, err := inv.NodeByName(ctx, node)
cidr, err := overlayRange(ctx, inv)
if err != nil {
return err
}
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
address, err := inv.AssignAddress(ctx, found.ID, cidr)
if err != nil {
return err
}
@@ -128,6 +176,10 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
" `module issue` them again and push (novox/hq issue 059)")
switch {
case adoptsTunnel:
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
len(tunnel.Peers))
case *hub:
fmt.Println(" the hub — every node not sharing a site routes through it")
case *endpoint == "":
@@ -154,15 +206,33 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
if err != nil {
return nil, err
}
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
tunnels, err := inv.Tunnels(ctx)
if err != nil {
return nil, err
}
carried, err := inv.CarriedPeers(ctx)
if err != nil {
return nil, err
}
nodes := make([]overlay.Node, 0, len(places))
for _, p := range places {
if !on[p.Name] {
continue
}
nodes = append(nodes, overlay.Node{
n := overlay.Node{
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
Site: p.Site, Hub: p.Hub, Address: p.Address,
})
}
if t, takes := tunnels[p.Name]; takes {
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config}
}
if p.Hub {
for _, c := range carried {
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
}
}
nodes = append(nodes, n)
}
if len(nodes) == 0 {
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
@@ -170,7 +240,11 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
// "no hub" to somebody who never asked for a network would be a lie about the cause.
return overlay.Empty(), nil
}
g, err := overlay.From(nodes, overlayCIDR(), "")
cidr, err := overlayRange(ctx, inv)
if err != nil {
return nil, err
}
g, err := overlay.From(nodes, cidr, "")
if g != nil {
// The artifact store, as this network reaches it. Found rather than configured: the
// provider is whichever module offers it, on whichever machine holds that module — and if
@@ -292,6 +366,17 @@ func overlayShow(ctx context.Context, open *stores) error {
return nil
}
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
// mesh until they enrol — and once one has, it is listed as the node it became.
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
if err != nil {
return err
}
carried, err := open.inventory.CarriedPeers(ctx)
if err != nil {
return err
}
for _, n := range nodes {
place := n.Address
if place == "" {
@@ -301,6 +386,12 @@ func overlayShow(ctx context.Context, open *stores) error {
}
fmt.Printf("%-16s %-14s", n.Name, place)
switch {
case n.Hub && adopted:
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
tunnel.Interface, tunnel.Range, tunnel.Port)
case n.Hub && hubName == n.Name && tunnel.Interface != "":
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's",
tunnel.Interface)
case n.Hub:
fmt.Print(" hub")
case !n.Reachable():
@@ -309,14 +400,35 @@ func overlayShow(ctx context.Context, open *stores) error {
if n.Site != "" {
fmt.Printf(" at %s", n.Site)
}
if n.TakesOver != nil && !n.Hub {
fmt.Printf(" takes over %s", n.TakesOver.Interface)
}
fmt.Println()
for _, p := range computed[n.Name] {
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
}
}
if len(carried) > 0 {
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
for _, c := range carried {
state := "not yet enrolled"
if c.EnrolledAs != "" {
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
}
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
}
}
return nil
}
// portOfEndpoint is the port in host:port, or empty.
func portOfEndpoint(endpoint string) string {
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
return endpoint[i+1:]
}
return ""
}
// SilentFor is how long a node may be quiet before the mesh says so.
//
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
+81
View File
@@ -108,3 +108,84 @@ func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
}
}
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
// the tunnel the hub holds — never stored anywhere else.
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
before, err := overlayRange(ctx, inv)
if err != nil || before != "10.99.0.0/16" {
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
}
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
// tunnel's key, and presenting the tunnel.
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
t.Fatal(err)
}
hub, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
const key = "THE-TUNNELS-KEY========================="
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
t.Fatal(err)
}
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
}); err != nil {
t.Fatal(err)
}
after, err := overlayRange(ctx, inv)
if err != nil || after != "192.0.2.0/24" {
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
}
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
// the tunnel's port.
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
if err == nil || !strings.Contains(err.Error(), "51900") {
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
}
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
t.Fatal(err)
}
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
t.Fatal(err)
}
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
}
// And the hub's declaration carries the peer and the takeover.
nodes, computed, err := graph(ctx, open)
if err != nil {
t.Fatal(err)
}
var hubNode overlay.Node
for _, n := range nodes {
if n.Name == "anchor" {
hubNode = n
}
}
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
}
carried := false
for _, p := range computed["anchor"] {
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
carried = true
}
}
if !carried {
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
}
}
+54 -12
View File
@@ -16,25 +16,64 @@ import (
// node's peer list to chase it, and an address that moves is the thing declaring the hub was
// meant to stop.
//
// Allocated in order from the range, taking the lowest free one. Not random: a person reading a
// peer list should be able to guess which node an address belongs to, and reuse of a released
// address is a smaller problem than a list nobody can hold in their head.
// **The adopted tunnel's addresses come first** (novox/hq ADR 0105). The hub that took over a
// tunnel is at the tunnel's own address. A node enrolling with a key the tunnel already routed
// to keeps the address the tunnel had for it — nothing a peer knows changes. And an address the
// tunnel holds for a peer that has not enrolled is never handed to anyone else: that peer is
// still reaching the hub at it.
//
// The rest is allocated in order from the range, taking the lowest free one. Not random: a person
// reading a peer list should be able to guess which node an address belongs to, and reuse of a
// released address is a smaller problem than a list nobody can hold in their head.
func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) {
prefix, err := netip.ParsePrefix(cidr)
if err != nil {
return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err)
}
var existing *string
var existing, key *string
var name string
var hub bool
if err := i.store.Pool().QueryRow(ctx,
`select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil {
`select name, host(overlay_address), overlay_key, is_hub from node where id = $1`, node).
Scan(&name, &existing, &key, &hub); err != nil {
return "", err
}
if existing != nil && *existing != "" {
return *existing, nil
}
tunnel, hubName, adopted, err := i.AdoptedTunnel(ctx)
if err != nil {
return "", err
}
if adopted && hub && hubName == name {
address, err := netip.ParsePrefix(tunnel.Address)
if err != nil {
return "", fmt.Errorf("the adopted tunnel's address %q: %w", tunnel.Address, err)
}
return i.place(ctx, node, address.Addr().String())
}
carried, err := i.CarriedPeers(ctx)
if err != nil {
return "", err
}
taken := map[string]bool{}
if adopted {
// The tunnel's own address is the hub's whether or not the hub has been placed yet.
if address, err := netip.ParsePrefix(tunnel.Address); err == nil {
taken[address.Addr().String()] = true
}
}
for _, p := range carried {
if key != nil && p.PublicKey == *key {
// The tunnel already routes to this key: the node keeps that address, and the peer
// notices nothing when its machine enrols.
return i.place(ctx, node, p.Address)
}
taken[p.Address] = true
}
rows, err := i.store.Pool().Query(ctx,
`select host(overlay_address) from node where overlay_address is not null`)
if err != nil {
@@ -58,12 +97,7 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
candidate := prefix.Masked().Addr().Next()
for prefix.Contains(candidate) {
if !taken[candidate.String()] {
if _, err := i.store.Pool().Exec(ctx,
`update node set overlay_address = $2::inet where id = $1`,
node, candidate.String()); err != nil {
return "", err
}
return candidate.String(), nil
return i.place(ctx, node, candidate.String())
}
candidate = candidate.Next()
}
@@ -72,5 +106,13 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
// halfway through assigning one node.
return "", fmt.Errorf(
"every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+
"range and renumbering it is a deliberate act", cidr, node)
"range and renumbering it is a deliberate act", cidr, name)
}
func (i *Inventory) place(ctx context.Context, node, address string) (string, error) {
if _, err := i.store.Pool().Exec(ctx,
`update node set overlay_address = $2::inet where id = $1`, node, address); err != nil {
return "", err
}
return address, nil
}
@@ -0,0 +1,27 @@
-- The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
--
-- On an adopted node that is the hub, the private network takes over the tunnel it finds: its
-- key, its port, its address and range, and every peer. The node presents what it found when it
-- enrols -- the same moment it presents its keys, because the found tunnel's key IS its key on the
-- private network from then on -- and the mesh composes every address from it.
-- What the node presented: interface, unit and configuration path, port, address and range, and
-- the tunnel's public key. The private key never travels; the node keeps it as its own overlay
-- key. Null on a node that found no tunnel, which is every converged one.
alter table node add column tunnel jsonb;
-- The node's last account of carrying it: the found interface down and disabled, the mesh's up
-- in its place. Null until the node says so.
alter table node add column tunnel_carried jsonb;
-- The peers the found tunnel had: a public key and the address the tunnel routed to it. Peers of
-- the tunnel, not nodes of the mesh, until they enrol -- a machine the mesh has no record of, whose
-- identity precedes its enrolment. One row per key, and one address per key on one tunnel.
create table tunnel_peer (
node uuid not null references node(id) on delete cascade,
public_key text not null,
address inet not null,
since timestamptz not null default now(),
primary key (node, public_key),
unique (node, address)
);
+313
View File
@@ -0,0 +1,313 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/netip"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
// The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
//
// On an adopted node that is the hub, the private network takes over the tunnel it finds: its
// private key, its port, its address and range, and every peer the found interface had. The node
// presents what it found when it enrols, in the same breath as its keys — the found tunnel's key is
// its key on the private network from then on — and the mesh composes every address from it: the
// hub's is the tunnel's, the range is the tunnel's, and a peer that enrols keeps the address the
// tunnel already had for its key.
// Tunnel is what a node found on its machine, as it presented it. No private key: the node keeps
// it as its own overlay key, sealed like any own secret, and the mesh records only the public half
// — which is then the node's overlay key too.
type Tunnel struct {
// Interface, Unit and Config are what the host takes over: the found interface, the unit
// that raised it, and its configuration file, which is kept like any held file.
Interface string `json:"interface"`
Unit string `json:"unit"`
Config string `json:"config"`
// Port is the port the found interface listened on — one the hosting provider already lets
// through, which is why it is worth taking.
Port int `json:"port"`
// Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the
// network that prefix names, 192.0.2.0/24.
Address string `json:"address"`
Range string `json:"range"`
// PublicKey is the found interface's, which every peer knows the tunnel by.
PublicKey string `json:"public_key"`
// Peers are the found interface's peers: each a public key and the address the tunnel routed
// to it.
Peers []TunnelPeer `json:"peers,omitempty"`
// At is when the node presented it; zero on a tunnel not yet recorded.
At time.Time `json:"at,omitempty"`
}
// TunnelPeer is one peer of a found tunnel.
type TunnelPeer struct {
PublicKey string `json:"public_key"`
// Address is the one host address the tunnel routed to the peer, without a prefix.
Address string `json:"address"`
}
// Carried is what a node last said about carrying the tunnel it found: the found interface down
// and disabled, the mesh's up in its place with the found key.
type Carried struct {
Interface string `json:"interface"`
Port int `json:"port"`
Range string `json:"range"`
Peers int `json:"peers"`
// Taken is whether the found interface is down and the mesh's up with its key; Kept is where
// the found configuration's original was kept.
Taken bool `json:"taken"`
Kept string `json:"kept,omitempty"`
At time.Time `json:"at"`
}
// CarriedPeer is one peer of the adopted tunnel as the mesh holds it: a peer of the tunnel, and
// — once a node enrols with that key — a node of the mesh as well.
type CarriedPeer struct {
PublicKey string
Address string
// EnrolledAs names the node that enrolled with this key, or is empty while none has.
EnrolledAs string
}
// ErrNoTunnel is asking about a tunnel on a node that presented none.
var ErrNoTunnel = errors.New("that node presented no tunnel")
// RecordTunnel keeps what a node presented, replacing what was there: the question is the tunnel
// as the node found it now. The peers are replaced whole for the same reason.
func (i *Inventory) RecordTunnel(ctx context.Context, nodeID string, t Tunnel) error {
if strings.TrimSpace(t.Interface) == "" || strings.TrimSpace(t.PublicKey) == "" {
return errors.New("a found tunnel names its interface and its public key, and this names neither")
}
if _, err := netip.ParsePrefix(t.Range); err != nil {
return fmt.Errorf("the found tunnel's range %q is not a range: %w", t.Range, err)
}
address, err := netip.ParsePrefix(t.Address)
if err != nil {
return fmt.Errorf("the found tunnel's address %q is not an address with a prefix: %w", t.Address, err)
}
peers := make([]TunnelPeer, 0, len(t.Peers))
for _, p := range t.Peers {
host, err := peerHost(p.Address)
if err != nil {
return fmt.Errorf("the found tunnel's peer %s: %w", shortKey(p.PublicKey), err)
}
if !address.Masked().Contains(host) {
return fmt.Errorf("the found tunnel's peer %s is routed at %s, outside the tunnel's %s",
shortKey(p.PublicKey), host, t.Range)
}
peers = append(peers, TunnelPeer{PublicKey: p.PublicKey, Address: host.String()})
}
t.Peers = nil
t.At = time.Now().UTC()
raw, err := json.Marshal(t)
if err != nil {
return err
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, `update node set tunnel = $2 where id = $1`, nodeID, raw); err != nil {
return err
}
if _, err := tx.Exec(ctx, `delete from tunnel_peer where node = $1`, nodeID); err != nil {
return err
}
for _, p := range peers {
if _, err := tx.Exec(ctx,
`insert into tunnel_peer (node, public_key, address) values ($1, $2, $3::inet)`,
nodeID, p.PublicKey, p.Address); err != nil {
return fmt.Errorf("recording the found tunnel's peer %s: %w", shortKey(p.PublicKey), err)
}
}
return tx.Commit(ctx)
}
// peerHost is the one host address a peer's allowed address names: a bare address, or a /32
// (or /128). A wider prefix is refused — a peer routed a whole range is not a machine with an
// address the mesh could give a node.
func peerHost(allowed string) (netip.Addr, error) {
allowed = strings.TrimSpace(allowed)
if a, err := netip.ParseAddr(allowed); err == nil {
return a, nil
}
p, err := netip.ParsePrefix(allowed)
if err != nil {
return netip.Addr{}, fmt.Errorf("%q is not an address", allowed)
}
if !p.IsSingleIP() {
return netip.Addr{}, fmt.Errorf("%q names a range, and a peer of the tunnel is one address", allowed)
}
return p.Addr(), nil
}
func shortKey(key string) string {
if len(key) > 8 {
return key[:8] + "…"
}
return key
}
// TunnelOf is the tunnel a node presented, with its peers, or ErrNoTunnel.
func (i *Inventory) TunnelOf(ctx context.Context, name string) (Tunnel, error) {
var raw []byte
var id string
err := i.store.Pool().QueryRow(ctx,
`select id, tunnel from node where name = $1`, name).Scan(&id, &raw)
if errors.Is(err, pgx.ErrNoRows) {
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
}
if err != nil {
return Tunnel{}, err
}
if len(raw) == 0 {
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoTunnel, name)
}
var t Tunnel
if err := json.Unmarshal(raw, &t); err != nil {
return Tunnel{}, err
}
t.Peers, err = i.tunnelPeers(ctx, id)
return t, err
}
func (i *Inventory) tunnelPeers(ctx context.Context, nodeID string) ([]TunnelPeer, error) {
rows, err := i.store.Pool().Query(ctx,
`select public_key, host(address) from tunnel_peer where node = $1 order by address`, nodeID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []TunnelPeer
for rows.Next() {
var p TunnelPeer
if err := rows.Scan(&p.PublicKey, &p.Address); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// AdoptedTunnel is the tunnel the mesh's private network runs over, if the hub adopted one: the
// hub's found tunnel, when the hub is adopted and its overlay key is the tunnel's. Absent, the mesh
// runs on its own range — and a hub that found a tunnel but holds another key did not adopt it,
// which `overlay show` says.
//
// The condition on the key is the condition of the whole record: a hub raised with a key of its
// own would drop every peer's packets on the found port (novox/hq ADR 0105, option 2), so the
// tunnel is adopted only when the hub answers to the key its peers know.
func (i *Inventory) AdoptedTunnel(ctx context.Context) (Tunnel, string, bool, error) {
var name string
var key *string
var adopted bool
err := i.store.Pool().QueryRow(ctx,
`select name, overlay_key, adopted from node where is_hub and tunnel is not null`).
Scan(&name, &key, &adopted)
if errors.Is(err, pgx.ErrNoRows) {
return Tunnel{}, "", false, nil
}
if err != nil {
return Tunnel{}, "", false, err
}
t, err := i.TunnelOf(ctx, name)
if err != nil {
return Tunnel{}, "", false, err
}
if !adopted || key == nil || *key != t.PublicKey {
return t, name, false, nil
}
return t, name, true, nil
}
// CarriedPeers is every peer of the adopted tunnel, with the node that enrolled under its key
// where one has: peers of the tunnel, and nodes of the mesh once they enrol. Empty when the hub
// adopted no tunnel.
func (i *Inventory) CarriedPeers(ctx context.Context) ([]CarriedPeer, error) {
rows, err := i.store.Pool().Query(ctx,
`select p.public_key, host(p.address), coalesce(n.name, '')
from tunnel_peer p
join node hub on hub.id = p.node and hub.is_hub and hub.adopted
and hub.tunnel is not null and hub.overlay_key = hub.tunnel->>'public_key'
left join node n on n.overlay_key = p.public_key
order by p.address`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []CarriedPeer
for rows.Next() {
var p CarriedPeer
if err := rows.Scan(&p.PublicKey, &p.Address, &p.EnrolledAs); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// Tunnels is every adopted node's found tunnel by node name, for the ones whose overlay key is the
// tunnel's — the ones whose private network takes it over. A found tunnel under another key is
// left running beside the mesh's, and ADR 0100's rule that the ranges differ applies to it.
func (i *Inventory) Tunnels(ctx context.Context) (map[string]Tunnel, error) {
rows, err := i.store.Pool().Query(ctx,
`select name, tunnel from node
where adopted and tunnel is not null and overlay_key = tunnel->>'public_key'`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]Tunnel{}
for rows.Next() {
var name string
var raw []byte
if err := rows.Scan(&name, &raw); err != nil {
return nil, err
}
var t Tunnel
if err := json.Unmarshal(raw, &t); err != nil {
return nil, err
}
out[name] = t
}
return out, rows.Err()
}
// RecordCarriedTunnel keeps what a node last said about carrying its found tunnel.
func (i *Inventory) RecordCarriedTunnel(ctx context.Context, nodeID string, c Carried) error {
c.At = time.Now().UTC()
raw, err := json.Marshal(c)
if err != nil {
return err
}
_, err = i.store.Pool().Exec(ctx, `update node set tunnel_carried = $2 where id = $1`, nodeID, raw)
return err
}
// CarriedTunnelOf is a node's last account of carrying its found tunnel, and whether it ever gave one.
func (i *Inventory) CarriedTunnelOf(ctx context.Context, name string) (Carried, bool, error) {
var raw []byte
err := i.store.Pool().QueryRow(ctx, `select tunnel_carried from node where name = $1`, name).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return Carried{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
}
if err != nil {
return Carried{}, false, err
}
if len(raw) == 0 {
return Carried{}, false, nil
}
var c Carried
if err := json.Unmarshal(raw, &c); err != nil {
return Carried{}, false, err
}
return c, true, nil
}
+175
View File
@@ -0,0 +1,175 @@
package inventory
import (
"errors"
"strings"
"testing"
)
// novox/hq ADR 0105: the mesh adopts the predecessor's tunnel in place. The controller reads the
// hub's address and range from the adopted tunnel, assigns an enrolling node the address its key
// already had, and refuses to hand out an address the tunnel already holds.
const (
tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key="
peerTwo = "PEER-KEY-two============================="
peerThree = "PEER-KEY-three==========================="
)
// theFoundTunnel is what a hub presents at enrolment: the predecessor's interface on a
// documentation range, with two peers each routed one address.
func theFoundTunnel() Tunnel {
return Tunnel{
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf",
Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey,
Peers: []TunnelPeer{{PublicKey: peerTwo, Address: "192.0.2.2/32"},
{PublicKey: peerThree, Address: "192.0.2.3"}},
}
}
// anAdoptedHub is an adopted node that enrolled with the found tunnel's key and presented the
// tunnel, then was placed as the hub — the order genesis does it in.
func anAdoptedHub(t *testing.T, inv *Inventory) Node {
t.Helper()
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
if err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(t.Context(), hub.ID, tunnelKey); err != nil {
t.Fatal(err)
}
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
t.Fatal(err)
}
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil {
t.Fatal(err)
}
return hub
}
func TestTheHubsAddressAndRangeComeFromTheAdoptedTunnel(t *testing.T) {
inv := fresh(t)
hub := anAdoptedHub(t, inv)
tunnel, name, adopted, err := inv.AdoptedTunnel(t.Context())
if err != nil {
t.Fatal(err)
}
if !adopted || name != "anchor" || tunnel.Range != "192.0.2.0/24" || tunnel.Port != 51900 {
t.Fatalf("the adopted tunnel did not read back: adopted=%t on %s, %+v", adopted, name, tunnel)
}
if len(tunnel.Peers) != 2 || tunnel.Peers[0].Address != "192.0.2.2" || tunnel.Peers[1].Address != "192.0.2.3" {
t.Fatalf("the peers did not read back as one host address each: %+v", tunnel.Peers)
}
// Whatever range the caller would allocate from, the hub is at the tunnel's own address.
address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16")
if err != nil {
t.Fatal(err)
}
if address != "192.0.2.1" {
t.Fatalf("the hub was given %s, not the address the tunnel it took over had", address)
}
}
func TestAnEnrollingNodeKeepsTheAddressTheTunnelHadForItsKey(t *testing.T) {
inv := fresh(t)
anAdoptedHub(t, inv)
// A predecessor machine enrols: its host took its found interface's key as its overlay key,
// which is the key the hub's tunnel already routes to.
peer, err := inv.AddNodeAs(t.Context(), "home-server", true)
if err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(t.Context(), peer.ID, peerThree); err != nil {
t.Fatal(err)
}
address, err := inv.AssignAddress(t.Context(), peer.ID, "192.0.2.0/24")
if err != nil {
t.Fatal(err)
}
if address != "192.0.2.3" {
t.Fatalf("the enrolling peer was given %s, not the 192.0.2.3 the tunnel had for its key", address)
}
carried, err := inv.CarriedPeers(t.Context())
if err != nil {
t.Fatal(err)
}
byKey := map[string]CarriedPeer{}
for _, c := range carried {
byKey[c.PublicKey] = c
}
if byKey[peerThree].EnrolledAs != "home-server" || byKey[peerTwo].EnrolledAs != "" {
t.Fatalf("the registry cannot say which peer is a node now: %+v", carried)
}
}
func TestAFreshNodeIsNeverGivenAnAddressTheTunnelHolds(t *testing.T) {
inv := fresh(t)
anAdoptedHub(t, inv)
// .1 is the hub, .2 and .3 are peers of the tunnel that have not enrolled: a new machine with
// a key of its own gets the next one, from the same range.
fresh, err := inv.AddNode(t.Context(), "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(t.Context(), fresh.ID, "A-KEY-OF-ITS-OWN========================"); err != nil {
t.Fatal(err)
}
address, err := inv.AssignAddress(t.Context(), fresh.ID, "192.0.2.0/24")
if err != nil {
t.Fatal(err)
}
if address != "192.0.2.4" {
t.Fatalf("a fresh node was given %s; 192.0.2.2 and .3 are the tunnel's peers and .1 its hub", address)
}
}
func TestATunnelUnderAnotherKeyIsNotAdopted(t *testing.T) {
// A hub whose overlay key is not the found tunnel's would drop every peer's packets on the
// found port (ADR 0105, option 2). Such a tunnel is recorded and not adopted: the mesh keeps
// its own range, and ADR 0100's non-overlap rule stands for it.
inv := fresh(t)
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
if err != nil {
t.Fatal(err)
}
if err := inv.RecordOverlayKey(t.Context(), hub.ID, "THE-MESHS-OWN-KEY======================="); err != nil {
t.Fatal(err)
}
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
t.Fatal(err)
}
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51820", "hosting", true, ""); err != nil {
t.Fatal(err)
}
if _, _, adopted, err := inv.AdoptedTunnel(t.Context()); err != nil || adopted {
t.Fatalf("a tunnel under another key was adopted (err %v)", err)
}
if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 0 {
t.Fatalf("peers of a tunnel that was not adopted are carried: %+v (err %v)", carried, err)
}
if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" {
t.Fatalf("the hub was given %s (err %v); it should allocate from the mesh's own range", address, err)
}
}
func TestAPeerRoutedARangeIsRefused(t *testing.T) {
inv := fresh(t)
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
if err != nil {
t.Fatal(err)
}
found := theFoundTunnel()
found.Peers = append(found.Peers, TunnelPeer{PublicKey: "WIDE", Address: "198.51.100.0/24"})
err = inv.RecordTunnel(t.Context(), hub.ID, found)
if err == nil || !strings.Contains(err.Error(), "names a range") {
t.Fatalf("a peer routed a whole range was recorded as a machine with an address: %v", err)
}
if _, err := inv.TunnelOf(t.Context(), "anchor"); !errors.Is(err, ErrNoTunnel) {
t.Fatalf("a refused tunnel was recorded anyway: %v", err)
}
}
+32
View File
@@ -124,6 +124,29 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
"%s's overlay key could not be recorded: %w", node.Name, err)
}
}
// And the tunnel it found, whose key is the overlay key above (novox/hq ADR 0105). Recorded
// before the token is spent for the same reason as the keys: the first declaration this node
// receives is composed from it, and a hub enrolled without its tunnel would be placed at an
// address of the mesh's choosing rather than the tunnel's.
if request.Tunnel != nil {
if request.Tunnel.PublicKey != request.OverlayKey {
return EnrolReply{}, fmt.Errorf("%s presented a tunnel under key %s and an overlay key "+
"that is not it; a tunnel is taken over with its own key or not at all", node.Name,
request.Tunnel.PublicKey)
}
peers := make([]inventory.TunnelPeer, 0, len(request.Tunnel.Peers))
for _, p := range request.Tunnel.Peers {
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
}
if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
Config: request.Tunnel.Config, Port: request.Tunnel.Port,
Address: request.Tunnel.Address, Range: request.Tunnel.Range,
PublicKey: request.Tunnel.PublicKey, Peers: peers,
}); err != nil {
return EnrolReply{}, fmt.Errorf("%s's found tunnel could not be recorded: %w", node.Name, err)
}
}
// Spent once the node is complete in the store.
if err := e.Inventory.Spend(ctx, secret, by); err != nil {
@@ -219,6 +242,15 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
return err
}
}
// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
if report.Tunnel != nil {
if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{
Interface: report.Tunnel.Interface, Port: report.Tunnel.Port, Range: report.Tunnel.Range,
Peers: report.Tunnel.Peers, Taken: report.Tunnel.Taken, Kept: report.Tunnel.Kept,
}); err != nil {
return err
}
}
// A bare word that a node is there is not an account of what the machine did or holds: it
// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
+42
View File
@@ -71,12 +71,39 @@ type EnrolRequest struct {
// node's public key could replay the spent token (novox/hq issue 083, on review).
Proof []byte `json:"proof,omitempty"`
// Tunnel is the tunnel this node found on its machine and whose key it took as its overlay
// key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented
// with the keys because it is one of them — OverlayKey above is this tunnel's public key when
// it is set — and the mesh composes the hub's address, the range and every carried peer from
// it. Nil from a node that found none, which is every converged one.
Tunnel *Tunnel `json:"tunnel,omitempty"`
// Redelivered is set by the control plane, never sent: the broker handed this request over a
// second time. Such a request does not finish an enrolment already spent — the first time may
// have answered, and the node holds what it was told.
Redelivered bool `json:"-"`
}
// Tunnel is a found tunnel as a node presents it: everything but its private key, which the node
// keeps as its own overlay key and never sends.
type Tunnel struct {
Interface string `json:"interface"`
Unit string `json:"unit"`
Config string `json:"config"`
Port int `json:"port"`
Address string `json:"address"`
Range string `json:"range"`
PublicKey string `json:"public_key"`
Peers []TunnelPeer `json:"peers,omitempty"`
}
// TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to
// it.
type TunnelPeer struct {
PublicKey string `json:"public_key"`
Address string `json:"address"`
}
// Signed is a declaration and the signature over it.
//
// The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies
@@ -129,6 +156,21 @@ type Report struct {
// Reachable is what can be reached on the machine now: every listening socket and every
// published container port. Only an adopted node reports it; it is what converging previews.
Reachable []Reach `json:"reachable,omitempty"`
// Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR
// 0105): the interface, its port, range and peer count, whether the found interface is down
// and the mesh's up in its place, and where the found configuration's original was kept.
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
}
// CarriedTunnel is a node's account of the tunnel it took over.
type CarriedTunnel struct {
Interface string `json:"interface"`
Port int `json:"port"`
Range string `json:"range"`
Peers int `json:"peers"`
Taken bool `json:"taken"`
Kept string `json:"kept,omitempty"`
}
// Held is one file or container found on an adopted node and kept as it was.
+35 -21
View File
@@ -43,6 +43,40 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
keyPath = DefaultKeyPath
}
up := Resource{
"id": "overlay-up", "type": "service", "unit": Unit,
"state": "running",
// Enabled, so the node comes back onto the network after a reboot without waiting to
// be told again. A node whose overlay only exists while something is watching is not
// a node that survives being switched off and on.
"boot": "enabled",
// And restarted when the peer list changes, because a running interface does not
// re-read its configuration.
//
// This is the whole of it: a node joins, every existing node's peer list changes,
// each file is replaced — and without this the service is already running, nothing
// reloads it, and every node keeps a network that no longer matches the mesh. It
// reports complete success. The lab found it the moment a third node arrived.
//
// Declared state rather than a command. The service must reflect the file; the host
// works out that it does not. A command to restart would be an action, and the link
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
// which is how this shape was arrived at.
"restart-on": []string{"overlay-config"},
}
if node.TakesOver != nil {
// The private network takes over the tunnel it found (novox/hq ADR 0105): before this
// unit starts, the host stops and disables the found one — never flushing it — and keeps
// its configuration like any held file. The key is already the found one: the node took
// it as its own overlay key when it enrolled, which is why the mesh's peer list for it
// carries the found peers under the key they know.
up["takes-over"] = map[string]any{
"interface": node.TakesOver.Interface,
"unit": node.TakesOver.Unit,
"config": node.TakesOver.Config,
}
}
resources := []Resource{
{
"id": "overlay-tools", "type": "package", "package": "wireguard-tools",
@@ -55,27 +89,7 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
"mode": "0600",
"content": config(node, peers, keyPath),
},
{
"id": "overlay-up", "type": "service", "unit": Unit,
"state": "running",
// Enabled, so the node comes back onto the network after a reboot without waiting to
// be told again. A node whose overlay only exists while something is watching is not
// a node that survives being switched off and on.
"boot": "enabled",
// And restarted when the peer list changes, because a running interface does not
// re-read its configuration.
//
// This is the whole of it: a node joins, every existing node's peer list changes,
// each file is replaced — and without this the service is already running, nothing
// reloads it, and every node keeps a network that no longer matches the mesh. It
// reports complete success. The lab found it the moment a third node arrived.
//
// Declared state rather than a command. The service must reflect the file; the host
// works out that it does not. A command to restart would be an action, and the link
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
// which is how this shape was arrived at.
"restart-on": []string{"overlay-config"},
},
up,
}
// The names used to be appended here, on the argument that a node with peers and no names is
+37
View File
@@ -223,3 +223,40 @@ func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) {
"compromised one could do")
}
}
// novox/hq ADR 0105: a node whose private network takes over the tunnel it found is told so on
// the interface's service, and nothing else about the declaration changes — the key is already
// the found one, taken at enrolment.
func TestTakingOverAFoundTunnelIsSaidOnTheInterfacesService(t *testing.T) {
node := Node{Name: "anchor", Key: "PUB", Address: "192.0.2.1", Hub: true,
Endpoint: "198.51.100.1:51900",
TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf"}}
config, resources := declarationFor(t, node, nil)
var up map[string]any
for _, r := range resources {
if r["type"] == "service" {
up = r
}
}
takes, ok := up["takes-over"].(map[string]any)
if !ok {
t.Fatalf("the interface's service does not say what it takes over: %+v", up)
}
if takes["unit"] != "wg-quick@wg0" || takes["config"] != "/etc/wireguard/wg0.conf" || takes["interface"] != "wg0" {
t.Errorf("the takeover names the wrong tunnel: %+v", takes)
}
if !strings.Contains(config, "ListenPort = 51900") {
t.Errorf("the hub's interface does not listen on the tunnel's port:\n%s", config)
}
if strings.Contains(config, "PrivateKey") {
t.Error("the found key travelled in the configuration; it is the node's own, set from its key file")
}
_, plain := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "192.0.2.4"}, nil)
for _, r := range plain {
if _, says := r["takes-over"]; says {
t.Error("a node taking over nothing was told to take something over")
}
}
}
+51
View File
@@ -26,6 +26,40 @@ type Node struct {
Site string
Hub bool
Address string
// Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the
// mesh has no record of, each known by the public key and the address the found tunnel routed
// to it. Only a hub has any. They stay in its peer list until a node enrols with that key —
// from then on the node is the peer.
Carried []Carried
// TakesOver names the found tunnel this node's private network replaces: its unit is stopped
// and disabled, never flushed, and its configuration kept, before the mesh's interface comes
// up with the found key. Nil on a node that raises the mesh's interface beside whatever it has.
TakesOver *TakeOver
}
// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a
// node of the mesh.
type Carried struct {
Key string
Address string
}
// TakeOver is the found tunnel a node's private network takes over, as the host is told it.
type TakeOver struct {
Interface string
Unit string
Config string
}
// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named
// by the key its packets arrive under.
func CarriedName(key string) string {
short := key
if len(short) > 8 {
short = short[:8] + "…"
}
return "a peer of the tunnel (" + short + ")"
}
// Reachable reports whether other nodes can dial this one. Declared, never inferred.
@@ -145,7 +179,9 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
// The hub holds every node that does not share a site with it, because those nodes
// route through it and it must know where to send the replies. Ones it cannot dial
// will dial it.
enrolled := map[string]bool{}
for _, other := range usable {
enrolled[other.Key] = true
if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
continue
}
@@ -156,6 +192,21 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
Why: "routes through this hub",
})
}
// And every peer of the tunnel it took over that has not enrolled (novox/hq ADR
// 0105): the same key and the same address the found tunnel had for it, so the
// machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in,
// as it always did. Once a node enrols with that key, the node's entry above is the
// peer, and WireGuard takes one entry per key.
for _, c := range self.Carried {
if enrolled[c.Key] {
continue
}
peers = append(peers, Peer{
Name: CarriedName(c.Key), Key: c.Key,
Allowed: c.Address + "/32",
Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh",
})
}
}
sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
+36
View File
@@ -304,3 +304,39 @@ func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) {
"nowhere to go")
}
}
// novox/hq ADR 0105: a hub that took over the predecessor's tunnel carries every peer that tunnel
// had, under the key and at the address the peer knows, until a node enrols with that key.
func TestTheHubCarriesTheTunnelsPeersUntilTheyEnrol(t *testing.T) {
hub := at("anchor", "hosting", "192.0.2.1", "198.51.100.1:51900", true)
hub.Carried = []Carried{
{Key: "key-home", Address: "192.0.2.2"},
{Key: "key-workstation", Address: "192.0.2.3"},
}
// The machine behind key-home enrolled: it is a node now, at the address it kept.
home := at("home", "house", "192.0.2.2", "", false)
home.Key = "key-home"
g, err := Compute([]Node{hub, home}, "192.0.2.0/24")
if err != nil {
t.Fatal(err)
}
peers := peersOf(t, g, "anchor")
carried, ok := peers[CarriedName("key-workstation")]
if !ok {
t.Fatalf("the hub does not carry the peer that has not enrolled: %+v", g["anchor"])
}
if carried.Allowed != "192.0.2.3/32" || carried.Endpoint != "" || carried.Key != "key-workstation" {
t.Errorf("a carried peer is not the tunnel's own entry — same key, its one address, no endpoint: %+v", carried)
}
if _, twice := peers[CarriedName("key-home")]; twice {
t.Error("a peer that enrolled is carried as well as listed as a node: WireGuard takes one entry per key")
}
if node, ok := peers["home"]; !ok || node.Key != "key-home" || node.Allowed != "192.0.2.2/32" {
t.Errorf("the enrolled peer is not the node it became: %+v", node)
}
// Carried peers are the hub's business only: a spoke routes everything through the hub.
if _, leaked := peersOf(t, g, "home")[CarriedName("key-workstation")]; leaked {
t.Error("a carried peer appeared in a spoke's peer list")
}
}
+89
View File
@@ -0,0 +1,89 @@
# Lab bed: the hub adopts the predecessor's tunnel (novox/hq ADR 0105)
A scenario and an integration-test skeleton for the mesh-lab repository, kept here because this
branch changes only the controller and the host. Move `adopt-the-tunnel.yml` to
`mesh-lab/scenarios/` and `adopt-the-tunnel.test.ts` to `mesh-lab/test/integration/` when the
feature lands; neither has been run. The skeleton follows `adoption.test.ts` and reuses its
harness. Documentation addresses throughout; the bed is node-agnostic.
## The bed, precisely
Three machines on one public segment, `hosting` (192.0.2.0/24), inbound allowed on all (the
anchor's firewall is the predecessor's, installed by the bed):
| machine | address | role |
|---|---|---|
| `anchor` | 192.0.2.10 | the machine in use: the predecessor's hub, then the mesh adopted on it |
| `peer-a` | 192.0.2.20 | a predecessor machine: reaches a service on the anchor through the tunnel; later **enrols and keeps its address** |
| `peer-b` | 192.0.2.30 | a second predecessor machine: reaches the same service; **never enrols** — the peer that must notice nothing throughout |
| `fresh` | 192.0.2.40 | a new machine: enrols later and **gets a fresh address from the same range** |
**Prepared the way the predecessor leaves a hub** (before genesis, by the bed, on `anchor`):
- `wireguard-tools` installed; a keypair made on each of `anchor`, `peer-a`, `peer-b`.
- `/etc/wireguard/wg0.conf` on the anchor: `[Interface]` `PrivateKey = <anchor's>`,
`ListenPort = 51900`, `Address = 10.10.0.1/24`; two `[Peer]` sections — `peer-a`'s public
key with `AllowedIPs = 10.10.0.2/32`, `peer-b`'s with `AllowedIPs = 10.10.0.3/32`. Raised with
`systemctl enable --now wg-quick@wg0`. **10.10.0.0/24 is deliberately not the mesh's default
range** (10.42.0.0/16), so a hub address in 10.10.0.0/24 can only have come from the tunnel.
- `wg0.conf` on each peer: its own key, `Address = 10.10.0.2/24` (resp. `.3/24`), one
`[Peer]` — the anchor's public key, `Endpoint = 192.0.2.10:51900`,
`AllowedIPs = 10.10.0.0/24`, `PersistentKeepalive = 25`. Raised the same way.
- A service on the anchor the peers reach **only over the tunnel**: a container publishing
`10.10.0.1:8081:80` (bound to the tunnel address, so a call from 192.0.2.20 to 10.10.0.1:8081
proves the tunnel carried it). Under a name no catalogue module uses — this bed is about the
tunnel, not about taking a service.
- The predecessor's firewall (`ufw`) allowing `51900/udp` and `22/tcp`, denying the rest — as ADR
0100's bed prepares it.
- A record of the anchor's `wg0` public key and of `sha256sum /etc/wireguard/wg0.conf`, taken
before genesis, for the assertions below.
**Genesis**, adopted, on the anchor: `mesh-bootstrap --adopted --node anchor --site hosting
--endpoint 192.0.2.10:51900 …` — no `--hub-port`, no `--overlay-range`, no `--tunnel`: the
installer finds `wg0` itself (one interface besides `mesh0`) and takes its port and range. The
bed asserts genesis **says** it found and took the tunnel.
## Assertions, in the record's order
- **T1 — the tunnel changes hands and the peers notice nothing.** After genesis and the push
that raises the private network on the anchor:
- `wg show interfaces` on the anchor lists `mesh0` and not `wg0`;
`systemctl is-active wg-quick@wg0` is inactive and `is-enabled` disabled;
- `/etc/wireguard/wg0.conf` is on disk with the recorded digest (kept, never flushed), and
`node show anchor` names where its original was kept;
- `wg show mesh0 public-key` is the anchor's recorded `wg0` public key; `wg show mesh0
listen-port` is 51900; `ip -o addr show dev mesh0` carries `10.10.0.1`; `wg show mesh0 peers`
lists both peers' public keys with their `/32` allowed addresses;
- a loop on `peer-a` and `peer-b` calling `http://10.10.0.1:8081/` every second, started before
genesis, records **no window of failure longer than one WireGuard re-handshake** (measure and
assert an upper bound — the switch is one unit stop plus one unit start on the anchor); the
peers' `wg0.conf` digests are unchanged; the peers' `wg show wg0 latest-handshakes` advance
after the switch.
- `overlay show` lists `anchor` as the hub over the tunnel it took over, and both peers under
"peers of the tunnel … not nodes of the mesh", not yet enrolled.
- **T2 — a peer enrols and keeps its address.** On `peer-a`: `node add peer-a --adopted`,
token issued, `mesh-host enrol --token …` **with the broker reached over the tunnel** (the
broker address in the token is `10.10.0.1:<bus>`, which only the tunnel routes); then `overlay
place peer-a --site house` and a push. Assert: `node show peer-a` says a tunnel `wg0` was
found and `overlay show` puts `peer-a` at **10.10.0.2**; the carried-peers list now says
`enrolled as peer-a`; the anchor's `mesh0` still has exactly one entry for `peer-a`'s key;
`peer-a`'s `wg0` is down and `mesh0` up with the same key; `peer-a` still reaches
`10.10.0.1:8081` and `peer-b` still does too, uninterrupted.
- **T3 — a new machine gets a fresh address from the same range.** `fresh` enrols converged
(no tunnel), is placed, pushed. Assert its address is **10.10.0.4** (`.1` hub, `.2` and `.3`
the tunnel's), that it reaches `10.10.0.1` (the hub) and `10.10.0.2` (the enrolled peer) —
`ping -c1` over `mesh0` — and that `peer-b`, never enrolled, is still served.
- **T4 — nothing derived from the address is stale.** `plan anchor --json` and `plan peer-a
--json` (and the rendered `/etc/hosts` on each node) name `10.10.0.1` for the anchor and
`10.10.0.2` for `peer-a`, and no address in `10.42.0.0/16`; the broker address handed to a
module issued on `peer-a` is `anchor.internal:<bus>` resolving to `10.10.0.1`; the same after a
second `push` of every node, byte for byte.
- **N — the narrowed ADR 0100 check.** On `fresh`, a converged genesis dry-run with
`--overlay-range 10.10.0.0/24` while a *second* tunnel the bed raises there (`wg1` at
10.10.0.9/24, not adopted because the node is converged) is up, is refused naming `wg1` —
the non-overlap rule still applies where a tunnel is not adopted.
## What is not asserted here
- Taking a service over the tunnel (ADR 0100's bed does that).
- IPv6 tunnels: the parser reads them, the bed prepares only IPv4.
@@ -0,0 +1,174 @@
/**
* THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). Skeleton — NOT YET RUN.
*
* The bed and every assertion are described in README.md beside this file; the numbered
* assertions here are that document's. Follows adoption.test.ts: same harness, same `on`/`must`
* helpers, same genesis wrapper.
*
* MESH_LAB_INCUS='sudo -n incus'
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
* MESH_LAB_CATALOG=.../mesh-catalog/modules
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, catalogueIsPresent } from "./harness.ts";
import { genesis } from "./genesis.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const skip = !capability.usable ? `lab not usable: ${capability.why}`
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
: catalogueIsPresent();
const SCENARIO = "adopt-the-tunnel";
const ANCHOR = "anchor", PEER_A = "peer-a", PEER_B = "peer-b", FRESH = "fresh";
const TUNNEL = { port: 51900, range: "10.10.0.0/24", hub: "10.10.0.1", a: "10.10.0.2", b: "10.10.0.3", fresh: "10.10.0.4" };
const SERVICE = `http://${TUNNEL.hub}:8081/`;
let instanceId = "";
let wg0Key = ""; // the anchor's wg0 public key, recorded before genesis
let wg0Digest = ""; // sha256 of /etc/wireguard/wg0.conf before genesis
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
const marker = stdout.lastIndexOf("__exit=");
if (marker < 0) return { out: stdout, ok: false };
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
const { out, ok } = await on(machine, command, timeoutMs);
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
return out;
}
/** The controller, a container on the anchor. */
async function control(args: string): Promise<string> {
return must(ANCHOR, `docker exec mesh-controller mesh-controller ${args}`);
}
/** Prepares a machine the way the predecessor leaves it: a keypair and a wg0 — see README.md. */
async function predecessorTunnelOn(machine: string, conf: (keys: Record<string, string>) => string, keys: Record<string, string>): Promise<void> {
await must(machine, "apt-get install -y wireguard-tools >/dev/null 2>&1 || pacman -S --noconfirm wireguard-tools >/dev/null");
await must(machine, `umask 077; printf '%s' '${conf(keys)}' > /etc/wireguard/wg0.conf`);
await must(machine, "systemctl enable --now wg-quick@wg0");
}
before(async () => {
if (skip) return;
await destroyAll(SCENARIO);
const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
instanceId = (await raise(scenario)).instanceId;
// Keys for the three predecessor machines, made where they live and never moved.
const keys: Record<string, string> = {};
for (const m of [ANCHOR, PEER_A, PEER_B]) {
await must(m, "umask 077; wg genkey > /etc/wireguard/predecessor.key");
keys[m] = (await must(m, "wg pubkey < /etc/wireguard/predecessor.key")).trim();
}
await predecessorTunnelOn(ANCHOR, k => [
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `ListenPort = ${TUNNEL.port}`, `Address = ${TUNNEL.hub}/24`,
"[Peer]", `PublicKey = ${k[PEER_A]}`, `AllowedIPs = ${TUNNEL.a}/32`,
"[Peer]", `PublicKey = ${k[PEER_B]}`, `AllowedIPs = ${TUNNEL.b}/32`,
].join("\n"), keys);
for (const [m, addr] of [[PEER_A, TUNNEL.a], [PEER_B, TUNNEL.b]] as const) {
await predecessorTunnelOn(m, k => [
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `Address = ${addr}/24`,
"[Peer]", `PublicKey = ${k[ANCHOR]}`, `Endpoint = 192.0.2.10:${TUNNEL.port}`, `AllowedIPs = ${TUNNEL.range}`, "PersistentKeepalive = 25",
].join("\n"), keys);
}
// A service reachable only over the tunnel, under a name no catalogue module uses.
await must(ANCHOR, `docker run -d --name predecessor-page -p ${TUNNEL.hub}:8081:80 nginx:alpine`);
// The predecessor's firewall: the tunnel's port and ssh, nothing else (as ADR 0100's bed).
await must(ANCHOR, `ufw --force reset >/dev/null; ufw default deny incoming; ufw allow 22/tcp; ufw allow ${TUNNEL.port}/udp; ufw --force enable`);
for (const m of [PEER_A, PEER_B]) assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service over the tunnel before genesis`);
wg0Key = (await must(ANCHOR, "wg show wg0 public-key")).trim();
wg0Digest = (await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0];
// The probe the peers keep running through the switch: one call a second, failures counted.
for (const m of [PEER_A, PEER_B]) await must(m, `nohup sh -c 'while :; do curl -fsS --max-time 1 ${SERVICE} >/dev/null 2>&1 || date +%s >> /tmp/failed; sleep 1; done' >/dev/null 2>&1 &`);
});
after(async () => { if (instanceId) await destroy(instanceId); });
test("T1 — adopted, the tunnel changes hands and the peers notice nothing", { skip }, async () => {
const ran = await genesis({ instanceId, machine: ANCHOR, node: ANCHOR, site: "hosting",
flags: ["--adopted", "--endpoint", `192.0.2.10:${TUNNEL.port}`] } as never);
assert.match(ran.said, /tunnel\s+wg0/i, `genesis did not say it found and took the tunnel:\n${ran.said}`);
const ifaces = await must(ANCHOR, "wg show interfaces");
assert.match(ifaces, /\bmesh0\b/); assert.doesNotMatch(ifaces, /\bwg0\b/);
assert.equal((await on(ANCHOR, "systemctl is-active wg-quick@wg0")).out.trim(), "inactive");
assert.equal((await on(ANCHOR, "systemctl is-enabled wg-quick@wg0")).out.trim(), "disabled");
assert.equal((await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0], wg0Digest, "the found configuration was changed or flushed");
assert.equal((await must(ANCHOR, "wg show mesh0 public-key")).trim(), wg0Key, "the mesh's interface is not up with the found key");
assert.equal((await must(ANCHOR, "wg show mesh0 listen-port")).trim(), String(TUNNEL.port));
assert.match(await must(ANCHOR, "ip -o addr show dev mesh0"), new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
const peers = await must(ANCHOR, "wg show mesh0 allowed-ips");
assert.match(peers, new RegExp(`${TUNNEL.a}/32`)); assert.match(peers, new RegExp(`${TUNNEL.b}/32`));
for (const m of [PEER_A, PEER_B]) {
const failed = (await on(m, "cat /tmp/failed 2>/dev/null | wc -l")).out.trim();
assert.ok(Number(failed) <= 5, `${m} lost the service for ${failed} seconds through the switch`);
assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service after the switch`);
}
const shown = await control("overlay show");
assert.match(shown, /anchor.*hub.*took over on wg0/);
assert.match(shown, /peers of the tunnel/); assert.match(shown, /not yet enrolled/);
assert.match(await control(`node show ${ANCHOR}`), /tunnel found\s+wg0 on port 51900/);
});
test("T2 — a peer enrols over the tunnel and keeps its address", { skip }, async () => {
await control(`node add ${PEER_A} --adopted`);
const token = (await control(`token issue --node ${PEER_A}`)).match(/token\s+(\S+)/)?.[1] ?? "";
// The token's broker address is the hub's tunnel address: only the tunnel routes it.
await must(PEER_A, `mesh-host enrol --token '${token}'`);
await control(`overlay place ${PEER_A} --site house`);
await control(`assign ${PEER_A} networking`);
await control(`push ${PEER_A} --wait 2m`);
assert.match(await control("overlay show"), new RegExp(`${PEER_A}\\s+${TUNNEL.a.replaceAll(".", "\\.")}`));
assert.match(await control("overlay show"), new RegExp(`enrolled as ${PEER_A}`));
const onHub = await must(ANCHOR, "wg show mesh0 allowed-ips");
assert.equal(onHub.split("\n").filter(l => l.includes(`${TUNNEL.a}/32`)).length, 1, "the enrolled peer's key appears twice on the hub");
assert.doesNotMatch(await must(PEER_A, "wg show interfaces"), /\bwg0\b/);
assert.ok((await on(PEER_A, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok, "the peer that never enrols lost the service");
});
test("T3 — a new machine gets a fresh address from the same range", { skip }, async () => {
await control(`node add ${FRESH}`);
const token = (await control(`token issue --node ${FRESH}`)).match(/token\s+(\S+)/)?.[1] ?? "";
await must(FRESH, `mesh-host enrol --token '${token}'`);
await control(`overlay place ${FRESH} --nothing`);
await control(`assign ${FRESH} networking`);
await control(`push ${FRESH} --wait 2m`);
assert.match(await control("overlay show"), new RegExp(`${FRESH}\\s+${TUNNEL.fresh.replaceAll(".", "\\.")}`));
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.hub}`)).ok, "the new machine does not reach the hub");
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.a}`)).ok, "the new machine does not reach the enrolled peer");
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
});
test("T4 — nothing derived from the address is stale", { skip }, async () => {
for (const n of [ANCHOR, PEER_A, FRESH]) {
const plan = await control(`plan ${n} --json`);
assert.doesNotMatch(plan, /10\.42\./, `${n}'s plan names the mesh's default range`);
assert.match(plan, new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
assert.match(await must(n, "cat /etc/hosts"), new RegExp(`${TUNNEL.hub.replaceAll(".", "\\.")}\\s+anchor\\.internal`));
}
const first = await control(`plan ${PEER_A} --json`);
await control("push");
assert.equal(await control(`plan ${PEER_A} --json`), first, "a push changed what the plan says");
});
test("N — where a tunnel is not adopted, the ranges must still differ (ADR 0100)", { skip }, async () => {
await must(FRESH, "umask 077; printf '[Interface]\\nPrivateKey = %s\\nAddress = 10.10.0.9/24\\n' \"$(wg genkey)\" > /etc/wireguard/wg1.conf; systemctl start wg-quick@wg1");
const ran = await genesis({ instanceId, machine: FRESH, node: FRESH, flags: ["--dry-run", "--overlay-range", TUNNEL.range], attempts: 1, verify: false, hostService: false } as never);
assert.match(ran.said, /wg1/, `a converged genesis did not refuse the overlapping tunnel it does not adopt:\n${ran.said}`);
});
+50
View File
@@ -0,0 +1,50 @@
# THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). See README.md beside this file.
#
# hosting (public)
# anchor 192.0.2.10 the predecessor's hub: wg0 on udp/51900, 10.10.0.1/24, two peers; then the
# mesh adopted on it, taking the tunnel over
# peer-a 192.0.2.20 a predecessor machine reaching a service on the anchor over the tunnel;
# enrols later and keeps 10.10.0.2
# peer-b 192.0.2.30 a predecessor machine that never enrols: must notice nothing, ever
# fresh 192.0.2.40 a new machine: enrols later and gets 10.10.0.4
#
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the bed.
scenario: adopt-the-tunnel
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 12GiB
cpus: 6
disk: 60GiB
peer-a:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
disk: 20GiB
peer-b:
at: { segment: hosting, address: [192.0.2.30] }
egress: true
inbound: allow
memory: 2GiB
cpus: 2
disk: 15GiB
fresh:
at: { segment: hosting, address: [192.0.2.40] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
disk: 20GiB
place:
all: [host, runtime]