novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Node-engine: the first read of the seat list is awaited on the reader's Ready, not on Start's return, and its end is told to the waiter and the composer (B9c, ADR 0636 §1; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 14:45:43 +01:00
parent f29b19aff2
commit c642471c16
10 changed files with 40 additions and 1385 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/nodeagent_integration_test.go
+22
View File
@@ -1019,3 +1019,25 @@ func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) {
t.Errorf("web does not stay as it ran after a start with the seat gone: %q", b.file("web"))
}
}
// The first read of the seat list ends the wait whether or not it changed what the reader held: Start returns at once, and
// the node waits on the reader's Ready before it says the read is over (ADR 0636 §1).
func TestNatsTheFirstReadOfTheSeatListEndsTheNodesWait(t *testing.T) {
clock := &fiveMinutes{}
was := seatWaitClock
seatWaitClock = clock
t.Cleanup(func() { seatWaitClock = was })
b := newItBedAs(t, nodegrant.For(itNode, "it2.mailer.domain"))
stop, _ := h8(t, b, false)
defer stop()
within(t, 30*time.Second, "the node did not end its wait at the first read of the seat list", func() bool {
st := seatListNow.Load()
if st == nil {
return false
}
<-st.reader.Ready()
waiting, _ := st.waiter.Waiting()
return st.reader.Status().Heard && !waiting
})
}
cmd/mesh-host/seatlist.go
+18 -5
View File
@@ -138,12 +138,13 @@ func (w findingsOfTheNode) Delete(f views.Finding) error {
// it holds again, saying so to the composer and to the waiter that writes the five-minute finding.
func startSeatList(ctx context.Context, session *link.Session, node string, say func(string)) (*seatListState, error) {
st := &seatListState{changed: make(chan struct{}, 1)}
var announce func()
// Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists.
var announce atomic.Pointer[func()]
reader, err := seatlist.Start(seatlist.Config{Bus: sessionBus{ctx, session}, Bootstrap: seatlist.Bootstrapped(),
OnChange: func(seats []catalogue.Seat) {
catalogue.UseSeats(seats)
if announce != nil { // the first read happens inside Start, before the waiter exists
announce()
if f := announce.Load(); f != nil { // the first read may end before the waiter exists
(*f)()
}
}})
if err != nil {
@@ -152,15 +153,27 @@ func startSeatList(ctx context.Context, session *link.Session, node string, say
st.reader = reader
st.waiter = &seatwait.Waiter{Node: node, Observer: "node-engine", Reader: reader, Findings: findingsOfTheNode{session},
Clock: seatWaitClock, Say: say}
announce = func() {
told := func() {
st.waiter.Changed()
select {
case st.changed <- struct{}{}:
default:
}
}
announce.Store(&told)
catalogue.UseSeats(reader.Seats())
announce()
told()
// The first read ends whether the seat answered or not, and a read that held what the reader held already says no
// change: its end is told to the waiter and the composer, so a status that moved from reading to heard starts or ends
// the bound (ADR 0636 §1). Start's return held nobody's start.
go func() {
select {
case <-reader.Ready():
catalogue.UseSeats(reader.Seats())
told()
case <-ctx.Done():
}
}()
seatListNow.Store(st)
return st, nil
}
internal/overlay/declaration.go
-187
View File
@@ -1,187 +0,0 @@
package overlay
import (
"encoding/json"
"fmt"
"strings"
)
// The overlay is delivered as an ordinary declaration.
//
// novox/hq 08-connectivity: what the host receives is an interface configuration and a peer list,
// as files. It does not compute them and it does not know what a private network is — the shapes
// it already has are enough, which is why connectivity needs nothing new from tier 0.
//
// **No private key travels.** The configuration points at a file the node wrote from a key the
// mesh has never seen, using WireGuard's own ability to set one after the interface is up. So the
// control plane composes a complete configuration for a node it cannot impersonate.
// Resource is one entry in a declaration, built here and read by the host.
type Resource map[string]any
// Declaration is what the mesh sends a node to put it on the network.
//
// Three resources and nothing clever: the tools, the configuration, and the interface running. A
// person can read it, which is the point — this is the first thing a node is ever told, and if it
// is wrong the node is unreachable and the mistake has to be findable by eye.
func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
if node.Address == "" {
return nil, fmt.Errorf("%s has no address on the overlay, so there is nothing to configure",
node.Name)
}
if keyPath == "" {
keyPath = DefaultKeyPath
}
up := Resource{
"id": "overlay-up", "type": "service", "unit": Unit,
"state": "running",
// Enabled, so the node comes back onto the network after a reboot without waiting to
// be told again. A node whose overlay only exists while something is watching is not
// a node that survives being switched off and on.
"boot": "enabled",
// And restarted when the peer list changes, because a running interface does not
// re-read its configuration.
//
// This is the whole of it: a node joins, every existing node's peer list changes,
// each file is replaced — and without this the service is already running, nothing
// reloads it, and every node keeps a network that no longer matches the mesh. It
// reports complete success. The lab found it the moment a third node arrived.
//
// Declared state rather than a command. The service must reflect the file; the host
// works out that it does not. A command to restart would be an action, and the link
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
// which is how this shape was arrived at.
"restart-on": []string{"overlay-config"},
}
if node.TakesOver != nil {
// The private network takes over the tunnel it found (novox/hq ADR 0105): before this
// unit starts, the host stops and disables the found one — never flushing it — and keeps
// its configuration like any held file. The key is already the found one: the node took
// it as its own overlay key when it enrolled, which is why the mesh's peer list for it
// carries the found peers under the key they know.
up["takes-over"] = map[string]any{
"interface": node.TakesOver.Interface,
"unit": node.TakesOver.Unit,
"config": node.TakesOver.Config,
}
}
resources := []Resource{
{
"id": "overlay-tools", "type": "package", "package": "wireguard-tools",
},
{
"id": "overlay-config", "type": "file", "path": ConfigPath,
// Readable only by root: it lists every peer's key and endpoint, which is a map of
// the mesh. Not secret in the way a private key is, and not something to leave
// world-readable on a machine somebody else also uses.
"mode": "0600",
"content": config(node, peers, keyPath),
},
up,
}
// The names used to be appended here, on the argument that a node with peers and no names is
// half on the network. True, and the wrong place to fix it: names would be identical over a
// different private network, so bundling them with WireGuard made one module out of two
// things. They are their own module now, requiring this one — which is what keeps them
// arriving together without pretending they are the same concern.
return json.Marshal(map[string]any{"declaration": 1, "resources": resources})
}
// config writes the interface file.
//
// Deliberately in the order a person would read it: who I am, then who I talk to, each with a
// line saying why it is there. A generated file that cannot be understood by the person it
// confuses is a generated file that gets edited by hand.
func config(node Node, peers []Peer, keyPath string) string {
var b strings.Builder
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever the\n")
b.WriteString("# peer graph changes, and an edit would survive until the next change and\n")
b.WriteString("# then vanish, which is worse than not being applied at all.\n")
fmt.Fprintf(&b, "#\n# node %s", node.Name)
if node.Site != "" {
fmt.Fprintf(&b, ", at %s", node.Site)
}
if !node.Reachable() {
b.WriteString(", not dialable — it opens every path itself")
}
b.WriteString("\n\n[Interface]\n")
fmt.Fprintf(&b, "Address = %s/32\n", node.Address)
if node.Reachable() {
if port := portOf(node.Endpoint); port != "" {
fmt.Fprintf(&b, "ListenPort = %s\n", port)
}
} else if node.TakesOver != nil && node.TakesOver.Port != 0 {
// Not dialable from the hub, but a LAN peer dials this node on the tunnel it took over,
// so the mesh's interface must listen on that same port (novox/hq: a taken tunnel brings
// its port). Without this the takeover guard refuses overlay-up, and re-placing the node
// with an endpoint — the guard's suggested remedy — breaks a NAT'd node's path.
fmt.Fprintf(&b, "ListenPort = %d\n", node.TakesOver.Port)
}
// The MTU the found tunnel carried, when it set one: a path tuned to 1380 (say) stalls TLS
// and hangs transfers if the mesh's interface comes up at the 1420 default, and no ping shows
// it (novox/hq: a taken tunnel carries its MTU).
if node.TakesOver != nil && node.TakesOver.MTU != 0 {
fmt.Fprintf(&b, "MTU = %d\n", node.TakesOver.MTU)
}
// The private key is set from a file the node wrote, so it never appears here and never
// travelled. Everything else in this file came from the mesh; this one line is the node's.
fmt.Fprintf(&b, "PostUp = wg set %%i private-key %s\n", keyPath)
if node.Hub {
// A hub carries traffic *between* its spokes, and a Linux machine does not forward
// packets unless it is told to. Without this every spoke reaches the hub perfectly and
// no spoke reaches any other — which is exactly how it failed in the lab, and it looks
// like a peering problem rather than a kernel setting.
//
// Here rather than in a separate resource because it is part of what being a hub means,
// and because it should last exactly as long as the interface does: a machine that stops
// being the hub should stop forwarding, and PostDown below is how that happens.
b.WriteString("PostUp = sysctl -q -w net.ipv4.ip_forward=1\n")
b.WriteString("PostDown = sysctl -q -w net.ipv4.ip_forward=0\n")
// And past the machine's own firewall, which on any node with a container runtime is
// closed. Docker sets the FORWARD policy to DROP and inserts its chains, so the foundation
// this mesh installs at tier 1 silently breaks the network it builds at tier 2: every
// spoke reaches the hub, no spoke reaches any other, and every part of it reports
// success. Found in the lab; nothing about it is visible from the mesh's own state.
//
// Inserted at the top so it precedes those chains, and removed on the way down so a
// machine that stops being the hub stops carrying other people's traffic. Guarded on
// iptables existing: a machine without it has no policy to get past.
for _, direction := range []string{"-i", "-o"} {
fmt.Fprintf(&b,
"PostUp = command -v iptables >/dev/null && iptables -I FORWARD 1 %s %%i -j ACCEPT || true\n",
direction)
}
for _, direction := range []string{"-i", "-o"} {
fmt.Fprintf(&b,
"PostDown = command -v iptables >/dev/null && iptables -D FORWARD %s %%i -j ACCEPT || true\n",
direction)
}
}
for _, p := range peers {
fmt.Fprintf(&b, "\n# %s — %s\n[Peer]\n", p.Name, p.Why)
fmt.Fprintf(&b, "PublicKey = %s\n", p.Key)
fmt.Fprintf(&b, "AllowedIPs = %s\n", p.Allowed)
if p.Endpoint != "" {
fmt.Fprintf(&b, "Endpoint = %s\n", p.Endpoint)
} else {
b.WriteString("# no endpoint: this peer cannot be dialled and opens the path itself\n")
}
if p.Keepalive {
b.WriteString("PersistentKeepalive = 25\n")
}
}
return b.String()
}
func portOf(endpoint string) string {
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
return endpoint[i+1:]
}
return ""
}
internal/overlay/declaration_test.go
-305
View File
@@ -1,305 +0,0 @@
package overlay
import (
"encoding/json"
"fmt"
"strings"
"testing"
)
func declarationFor(t *testing.T, node Node, peers []Peer) (string, []map[string]any) {
t.Helper()
raw, err := Declaration(node, peers, "")
if err != nil {
t.Fatal(err)
}
var d struct {
Declaration int `json:"declaration"`
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(raw, &d); err != nil {
t.Fatal(err)
}
if d.Declaration != 1 {
t.Fatalf("declaration version %d", d.Declaration)
}
for _, r := range d.Resources {
if r["type"] == "file" {
return r["content"].(string), d.Resources
}
}
t.Fatal("the declaration has no configuration file in it")
return "", nil
}
func TestNoPrivateKeyEverTravels(t *testing.T) {
// The property the whole design rests on: the node generated its keypair and kept the private
// half, so the mesh composes a configuration for a node it cannot impersonate. A private key
// appearing here would mean the control plane had one — and a copy of its database would then
// be every node's network identity.
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"},
[]Peer{{Name: "anchor", Key: "HUB", Allowed: "10.42.0.0/16", Endpoint: "198.51.100.1:51820"}})
if strings.Contains(config, "PrivateKey") {
t.Error("the configuration carries a PrivateKey line; the mesh must never hold one")
}
if !strings.Contains(config, "private-key "+DefaultKeyPath) {
t.Error("the configuration does not point at the key file the node wrote, so the " +
"interface would come up with no key at all")
}
}
func TestTheDeclarationUsesOnlyShapesTheHostAlreadyHas(t *testing.T) {
// Connectivity needs nothing new from tier 0, and that is worth holding: the host does not
// know what a private network is, and should not learn.
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
allowed := map[string]bool{"package": true, "file": true, "service": true,
"directory": true, "container": true, "action": true}
for _, r := range resources {
if !allowed[r["type"].(string)] {
t.Errorf("the overlay declaration uses %q, which the host does not have", r["type"])
}
}
}
func TestTheInterfaceComesBackAfterAReboot(t *testing.T) {
// A node whose overlay only exists while something is watching is not a node that survives
// being switched off and on — and it would come back unreachable, which is the worst way to
// come back.
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
for _, r := range resources {
if r["type"] == "service" {
if r["boot"] != "enabled" {
t.Error("the overlay interface is not enabled at boot")
}
if r["state"] != "running" {
t.Error("the overlay interface is not asked to be running")
}
return
}
}
t.Error("nothing in the declaration brings the interface up")
}
func TestTheConfigurationIsNotWorldReadable(t *testing.T) {
// It lists every peer's key and endpoint, which is a map of the mesh. Not secret the way a
// private key is, and not something to leave readable on a machine somebody else also uses.
// The peer list specifically, not every file. `/etc/hosts` is in here too and must be
// world-readable, or nothing on the machine resolves anything — a check that swept all files
// would force it to 0600 and break the machine to protect a file that is not secret.
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
for _, r := range resources {
if r["id"] == "overlay-config" && r["mode"] != "0600" {
t.Errorf("the peer list is mode %v", r["mode"])
}
if r["id"] == "mesh-names" && r["mode"] != "0644" {
t.Errorf("the name file is mode %v; nothing on the machine could read it", r["mode"])
}
}
}
func TestAPeerThatCannotBeDialledSaysSo(t *testing.T) {
// A [Peer] with no Endpoint is correct and looks like a mistake. Saying why stops somebody
// helpfully adding one that cannot work.
config, _ := declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
Endpoint: "198.51.100.1:51820", Hub: true},
[]Peer{{Name: "laptop", Key: "PUB", Allowed: "10.42.0.2/32", Why: "routes through this hub"}})
if strings.Contains(config, "Endpoint =") {
t.Error("an endpoint was written for a peer that has none")
}
if !strings.Contains(config, "cannot be dialled") {
t.Error("the file does not say why that peer has no endpoint")
}
}
func TestTheFileSaysNotToEditIt(t *testing.T) {
// It is replaced whenever the graph changes. An edit survives until then and vanishes, which
// is worse than never being applied — the machine works, then stops, and nothing changed
// that anybody remembers.
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
if !strings.Contains(config, "Do not edit") {
t.Error("a generated file does not say it is generated")
}
}
func TestANodeWithNoAddressIsRefused(t *testing.T) {
// Rather than a configuration with a blank address, which wg-quick would reject on the
// machine, at boot, where the failure is much harder to see.
if _, err := Declaration(Node{Name: "laptop", Key: "PUB"}, nil, ""); err == nil {
t.Fatal("a node with no overlay address was given a configuration")
}
}
func TestOnlyAReachableNodeListens(t *testing.T) {
// A ListenPort on a node nothing can dial is a port open for no reason.
config, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
if strings.Contains(config, "ListenPort") {
t.Error("a node that cannot be dialled was told to listen")
}
config, _ = declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
Endpoint: "198.51.100.1:51820"}, nil)
if !strings.Contains(config, "ListenPort = 51820") {
t.Error("a reachable node does not listen on the port its endpoint names")
}
}
func TestTheServiceIsRestartedWhenThePeerListChanges(t *testing.T) {
// The fault this exists to catch, found in the lab the moment a third node arrived: a running
// WireGuard interface does not re-read its configuration. The file was replaced, the service
// was already running so nothing reloaded it, and every existing node kept a network that no
// longer matched the mesh — while looking entirely successful.
//
// So the declaration has to verify what the interface is *carrying*, not what the file says.
//
// And it must be declared state rather than a command: the host refuses an action arriving
// over the link (novox/hq ADR 0005), correctly, which is how this shape was arrived at. There
// is a test below that no action is ever in here.
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"},
[]Peer{{Name: "anchor", Key: "HUBKEY", Allowed: "10.42.0.0/16"}})
for _, r := range resources {
if r["type"] != "service" {
continue
}
reflects := fmt.Sprint(r["restart-on"])
if !strings.Contains(reflects, "overlay-config") {
t.Errorf("the interface does not restart when its configuration changes: %v", reflects)
}
return
}
t.Error("nothing in the declaration brings the interface up")
}
func TestTheOverlayDeclarationCarriesNoAction(t *testing.T) {
// The link may not carry an action, and the host refuses a declaration containing one — whole,
// not in part. An overlay declaration with an action in it does not half-apply: it leaves the
// node with no network at all.
_, resources := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
for _, r := range resources {
if r["type"] == "action" {
t.Errorf("the declaration contains an action (%v); the host will refuse the whole "+
"thing and the node will have no network", r["id"])
}
}
}
func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) {
// A hub carries traffic between its spokes, and Linux does not forward packets unless told
// to. Without it every spoke reaches the hub perfectly and no spoke reaches any other — which
// is how it failed in the lab, and it presents as a peering problem rather than a kernel
// setting, so it is worth being sure of.
hub, _ := declarationFor(t, Node{Name: "anchor", Key: "HUB", Address: "10.42.0.1",
Endpoint: "198.51.100.1:51820", Hub: true}, nil)
if !strings.Contains(hub, "ip_forward=1") {
t.Error("the hub does not enable forwarding, so its spokes cannot reach each other")
}
if !strings.Contains(hub, "ip_forward=0") {
t.Error("the hub never stops forwarding; a machine that stops being the hub would keep " +
"passing traffic it is no longer part of")
}
// And past the machine's own firewall. Any node with a container runtime has FORWARD set to
// DROP by Docker, so enabling ip_forward alone changes nothing — which is exactly how it
// failed, with every spoke reaching the hub and no spoke reaching any other.
// Checked as PostUp specifically. "contains FORWARD" passes on the PostDown line alone,
// which would leave a hub that tears down rules it never put up.
if !strings.Contains(hub, "PostUp = command -v iptables") {
t.Error("the hub does not open its own firewall, so a container runtime's DROP policy " +
"silently eats everything it was supposed to carry")
}
if !strings.Contains(hub, "PostDown = command -v iptables") {
t.Error("the hub never removes those rules, so a machine that stops being the hub keeps " +
"passing traffic it is no longer part of")
}
spoke, _ := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "10.42.0.2"}, nil)
if strings.Contains(spoke, "FORWARD") {
t.Error("a spoke was given forwarding rules it has no use for")
}
if strings.Contains(spoke, "ip_forward") {
t.Error("a spoke was told to forward packets, which is not its job and widens what a " +
"compromised one could do")
}
}
// novox/hq ADR 0105: a node whose private network takes over the tunnel it found is told so on
// the interface's service, and nothing else about the declaration changes — the key is already
// the found one, taken at enrolment.
func TestTakingOverAFoundTunnelIsSaidOnTheInterfacesService(t *testing.T) {
node := Node{Name: "anchor", Key: "PUB", Address: "192.0.2.1", Hub: true,
Endpoint: "198.51.100.1:51900",
TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf"}}
config, resources := declarationFor(t, node, nil)
var up map[string]any
for _, r := range resources {
if r["type"] == "service" {
up = r
}
}
takes, ok := up["takes-over"].(map[string]any)
if !ok {
t.Fatalf("the interface's service does not say what it takes over: %+v", up)
}
if takes["unit"] != "wg-quick@wg0" || takes["config"] != "/etc/wireguard/wg0.conf" || takes["interface"] != "wg0" {
t.Errorf("the takeover names the wrong tunnel: %+v", takes)
}
if !strings.Contains(config, "ListenPort = 51900") {
t.Errorf("the hub's interface does not listen on the tunnel's port:\n%s", config)
}
if strings.Contains(config, "PrivateKey") {
t.Error("the found key travelled in the configuration; it is the node's own, set from its key file")
}
_, plain := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "192.0.2.4"}, nil)
for _, r := range plain {
if _, says := r["takes-over"]; says {
t.Error("a node taking over nothing was told to take something over")
}
}
}
func TestATakenTunnelBringsItsListenPortEvenWhenNotDialable(t *testing.T) {
// A home node behind NAT (no Endpoint, so not Reachable) that took over a tunnel must still
// listen on that tunnel's port, because its LAN peers dial it there (novox/hq: a taken tunnel
// brings its port). Without this the takeover guard refuses overlay-up.
config, _ := declarationFor(t, Node{
Name: "shanks", Key: "SPOKE", Address: "10.10.0.3",
TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Port: 51820},
}, nil)
if !strings.Contains(config, "ListenPort = 51820") {
t.Fatalf("a taken tunnel's port must be the mesh interface's ListenPort:\n%s", config)
}
if strings.Contains(config, "Endpoint =") {
t.Error("a node that only listens for LAN peers must not advertise an endpoint")
}
}
func TestANodeWithNoTunnelAndNoEndpointStillListensOnNothing(t *testing.T) {
// The guard against over-emitting: a plain spoke with neither an endpoint nor a taken tunnel
// writes no ListenPort — it purely dials out.
config, _ := declarationFor(t, Node{Name: "laptop", Key: "K", Address: "10.10.0.9"}, nil)
if strings.Contains(config, "ListenPort") {
t.Fatalf("a dial-only node needs no ListenPort:\n%s", config)
}
}
// A path tuned to a smaller MTU (1380 here) survives the takeover: the mesh interface comes up with the same MTU, or
// transfers hang silently (novox/hq: a taken tunnel carries its MTU). A tunnel that set none gets no MTU line.
func TestATakenTunnelCarriesItsMTU(t *testing.T) {
for _, c := range []struct {
mtu int
want string
}{{1380, "MTU = 1380"}, {0, ""}} {
config, _ := declarationFor(t, Node{
Name: "shanks", Key: "SPOKE", Address: "10.10.0.3",
TakesOver: &TakeOver{Interface: "wg0", Port: 51820, MTU: c.mtu},
}, nil)
if c.want != "" && !strings.Contains(config, c.want) || c.want == "" && strings.Contains(config, "MTU") {
t.Errorf("a tunnel of MTU %d gives:\n%s", c.mtu, config)
}
}
}
internal/overlay/generator.go
-151
View File
@@ -1,151 +0,0 @@
package overlay
import (
"encoding/json"
"fmt"
"strconv"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The private network as a module rather than as code beside the module system.
//
// A machine's peer list is derived from every other machine, so it differs on each one and
// changes when any of them changes — there is nothing that could be written in a manifest. So the
// module says its resources are computed, and this computes them.
//
// What that buys, beyond one mechanism instead of two: **a machine is on the private network
// because it was assigned the module.** Before this, every machine with a key and an address was
// on it, and there was no way to say a machine should stay off.
//
// It is worth saying what is *not* here, because networking looks like it should be foundational.
// The host needs none of this. It has an address and a route to the broker before the mesh exists
// — that is the machine's own networking — and the broker's address is carried in the enrolment
// token rather than resolved. So the private network is something the mesh installs on top, like
// anything else, and a node without it is an ordinary node that nothing reaches directly.
// **A resolver's data went the same way as the names.** Two constants used to sit here — a
// `mesh-resolver` module that would write the machines as wildcards, and a `resolver-data`
// requirement a daemon would ask for. Nothing ever provided or consumed either: a module that
// answers names asks for the `node-zones` fact in its own manifest (catalogue.FactsInto) and
// requires Addressing, since the file is made of the mesh's addresses and means nothing off the
// network. A requirement nothing provides is refused at resolution, so leaving the names here
// would only have documented a mechanism that does not exist.
// **There is no bundle any more** (novox/hq ADR 0226). A `networking` module requiring this one and
// nothing else used to be what a machine was assigned, so that "get the network working" was one
// word and a second VPN could be chosen by assigning it instead. The mesh has one private network,
// every machine is on it, and the bundle was a second name for this module that every machine
// carried. A machine is assigned Name directly; another VPN is still chosen by assigning it in its
// place, which is all the bundle ever did.
// Generator answers what one node's network configuration is.
type Generator struct {
nodes []Node
graph Graph
// keyPath is where each node keeps the private half it generated. Named rather than carried:
// the mesh has never seen it and never will.
keyPath string
// No registry. Being on the network is still what grants a machine the right to pull from the
// mesh's artifact store in the clear (novox/hq ADR 0082), but the runtime's file is the runtime's
// module's: the private network writes nothing into it, and that module states the registry
// itself, told where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR
// 0222, issue 190).
}
// From builds a generator over the machines that are part of the network.
//
// The nodes given are the ones assigned the module — not every node the mesh knows. A machine
// that was never given it is absent from everybody's peer list and from the names, which is what
// "not on the network" has to mean.
func From(nodes []Node, cidr, keyPath string) (*Generator, error) {
graph, err := Compute(nodes, cidr)
if err != nil {
return nil, err
}
return &Generator{nodes: nodes, graph: graph, keyPath: keyPath}, nil
}
// Resources is one node's interface, peers and names.
func (g *Generator) Resources(node string) ([]map[string]any, bool, error) {
peers, part := g.graph[node]
if !part {
// Assigned and not yet placed on the network. Ordinary and brief, so it is an answer
// rather than an error.
return nil, false, nil
}
var self Node
for _, n := range g.nodes {
if n.Name == node {
self = n
}
}
raw, err := Declaration(self, peers, g.keyPath)
if err != nil {
return nil, false, err
}
var parsed struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, false, err
}
return parsed.Resources, true, nil
}
// Nodes are the machines this generator was built over, so a caller can say who is on the network.
func (g *Generator) Nodes() []Node { return g.nodes }
// Graph is the peer list per node, for showing.
func (g *Generator) Graph() Graph { return g.graph }
// Empty is a network nobody is on.
//
// A mesh where no machine was given the module. Legitimate rather than broken — every node still
// reaches the broker, which is what being in the mesh is — so it answers "not part of this" for
// everyone instead of refusing for want of a hub.
func Empty() *Generator { return &Generator{graph: Graph{}} }
// Listens is the port this node accepts the private network on, which only a hub has.
//
// **A fact about this machine's place in the mesh, not about the module.** Every machine on the
// network runs the same module; a hub is dialled by every node at other sites and needs its port
// open, and a machine that is not a hub dials out and needs nothing open at all. A static field in
// a manifest is one answer for every machine that runs it, so it cannot say this — and the machine
// it would get wrong is the one facing the public internet, which is the machine that most needs
// filtering.
//
// The port is the one in the endpoint, which is also where the interface takes its ListenPort
// from. One source, so a rule set cannot open a port the interface is not on.
func (g *Generator) Listens(node string) ([]catalogue.Listening, error) {
for _, n := range g.nodes {
if n.Name != node {
continue
}
if !n.Reachable() {
// It dials out and nothing dials it. Opening a port here would be opening one on a
// machine nothing connects to, which is not harmless — it is a rule with no source
// that somebody later has to work out the reason for.
return nil, nil
}
port := portOf(n.Endpoint)
if port == "" {
return nil, fmt.Errorf(
"%s is reachable at %q and no port can be read from it, so what it must accept "+
"the private network on is unknown", node, n.Endpoint)
}
number, err := strconv.Atoi(port)
if err != nil {
return nil, fmt.Errorf("%s is reachable at %q, and %q is not a port", node, n.Endpoint, port)
}
return []catalogue.Listening{{
Port: number, Protocol: "udp", From: catalogue.FromEverywhere,
// From everywhere, and deliberately: a node at another site is not on the private
// network until this port lets it on, so restricting this to the mesh would be a
// rule that can never be satisfied by the thing it exists for.
Why: "the private network — a node at another site has no other way in",
}}, nil
}
return nil, nil
}
internal/overlay/generator_test.go
-30
View File
@@ -1,30 +0,0 @@
package overlay
import (
"testing"
)
// novox/hq ADR 0222, issue 190: the runtime's file and service are the runtime's module's. The
// private network writes nothing into either — being on it still grants the right to pull from the
// mesh's store in the clear (ADR 0082), and the runtime's module states that trust itself.
func TestTheNetworkWritesNothingOfTheRuntimes(t *testing.T) {
nodes := []Node{
{Name: "anchor", Site: "lab", Hub: true, Endpoint: "192.0.2.10:51820", Key: "k1", Address: "10.42.0.1"},
{Name: "node2", Site: "lab", Key: "k2", Address: "10.42.0.2"},
}
g, err := From(nodes, "10.42.0.0/16", "")
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "node2"} {
resources, part, err := g.Resources(node)
if err != nil || !part {
t.Fatalf("%s: resources: %v part=%v", node, err, part)
}
for _, r := range resources {
if r["path"] == "/etc/docker/daemon.json" || r["unit"] == "docker.service" {
t.Errorf("%s: the private network declares the runtime's %v", node, r)
}
}
}
}
internal/overlay/graph.go
-231
View File
@@ -1,231 +0,0 @@
// Package overlay computes the private network every node runs on.
//
// novox/hq 08-connectivity. This is control-plane work by definition: a peer list is derived from
// every node at once, and no node has that. A node computes nothing about the mesh — it generates
// a keypair, publishes the public half, and receives the rest.
//
// The shape is a hub, with direct peering between nodes at the same site. Not a full mesh, and
// the reason is a property of WireGuard rather than a preference: there is no failover. A more
// specific route to a dead endpoint blackholes; it does not fall back to the general one. So a
// node gets exactly one path to any peer, because two would mean one of them silently swallowing
// traffic.
package overlay
import (
"errors"
"fmt"
"sort"
"strings"
)
// Node is one machine's place on the network, as the mesh holds it.
type Node struct {
Name string
Key string
Endpoint string
Site string
Hub bool
Address string
// Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the
// mesh has no record of, each known by the public key and the address the found tunnel routed
// to it. Only a hub has any. They stay in its peer list until a node enrols with that key —
// from then on the node is the peer.
Carried []Carried
// TakesOver names the found tunnel this node's private network replaces: its unit is stopped
// and disabled, never flushed, and its configuration kept, before the mesh's interface comes
// up with the found key. Nil on a node that raises the mesh's interface beside whatever it has.
TakesOver *TakeOver
}
// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a
// node of the mesh.
type Carried struct {
Key string
Address string
}
// TakeOver is the found tunnel a node's private network takes over, as the host is told it.
type TakeOver struct {
Interface string
Unit string
Config string
// MTU is the found tunnel's, when it set one — emitted so a tuned path keeps its MTU.
MTU int
// Port is the port the found tunnel listened on. The mesh's interface must listen on it too,
// even on a node that is not dialable from the hub: a home node's LAN peers dial it there
// (novox/hq: a taken tunnel brings its port). Listening is "a peer dials me here"; it is not
// "the hub can dial me", which is Reachable — the two were conflated.
Port int
}
// HostPrefix is one address as a route: /32 for IPv4, /128 for IPv6.
func HostPrefix(address string) string {
if strings.Contains(address, ":") {
return address + "/128"
}
return address + "/32"
}
// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named
// by the key its packets arrive under.
func CarriedName(key string) string {
short := key
if len(short) > 8 {
short = short[:8] + "…"
}
return "a peer of the tunnel (" + short + ")"
}
// Reachable reports whether other nodes can dial this one. Declared, never inferred.
func (n Node) Reachable() bool { return strings.TrimSpace(n.Endpoint) != "" }
// Peer is one entry in a node's peer list.
type Peer struct {
Name string
Key string
// Endpoint is empty when this peer cannot be dialled — it must dial us instead.
Endpoint string
// Allowed is what traffic goes down this tunnel. A single address for a direct peer; the
// whole overlay for the hub, which is what makes it the route of last resort.
Allowed string
// Keepalive matters only on the side behind NAT: a node that cannot be dialled has to keep
// the path open from its end, or the peer's first packet arrives at a mapping that has
// already expired.
Keepalive bool
// Why this peer is in the list, for a person reading a generated file and wondering.
Why string
}
// Graph is every node's peer list.
type Graph map[string][]Peer
// ErrNoHub means nobody has said which node is the hub.
//
// Its own error rather than an empty graph: a mesh with no hub has no path between sites, and
// answering with "no peers" would look like a working mesh where nothing can reach anything.
var ErrNoHub = errors.New("this mesh has no hub, so there is no path between sites")
// Compute derives every node's peer list.
//
// Nodes without a key or an address are skipped rather than refused: a node that has enrolled and
// not yet been given a place on the network is an ordinary in-between state, and failing the whole
// graph because one node is half-configured would mean no node gets a network.
func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
var hub *Node
usable := make([]Node, 0, len(nodes))
for i := range nodes {
n := nodes[i]
if n.Key == "" || n.Address == "" {
continue
}
usable = append(usable, n)
if n.Hub {
hub = &usable[len(usable)-1]
}
}
if len(usable) == 0 {
return Graph{}, nil
}
if hub == nil {
return nil, ErrNoHub
}
if !hub.Reachable() {
return nil, fmt.Errorf(
"%s is the hub and has no endpoint, so nothing can dial it. The hub is the one node "+
"that must be reachable from wherever the others are", hub.Name)
}
graph := Graph{}
for _, self := range usable {
var peers []Peer
// A node may share a site with the hub, and then the hub is one peer rather than two.
// Written before the loop because it changes what that loop may emit: WireGuard takes one
// entry per public key, so a hub appearing twice is a configuration it refuses — and the
// mesh would have produced it silently. The lab found this on the first two machines that
// shared a site with their hub.
hubIsHere := !self.Hub && self.Site != "" && self.Site == hub.Site
for _, other := range usable {
if other.Name == self.Name || (hubIsHere && other.Name == hub.Name) {
continue
}
// Two nodes at the same site peer directly — but only if one of them can be
// dialled. If neither can, nobody opens the path, and the direct route is more
// specific than the hub's, so it wins and blackholes. That is this design's own
// stated hazard arriving in it: *a more specific route to a dead endpoint
// blackholes; it does not fall back to the general one.*
//
// Found in the lab with two machines at one site behind no reachable address, which
// is the ordinary shape of a home: they were given each other as peers, neither
// could start, and they could not reach each other at all while both reached the hub
// perfectly.
if self.Site != "" && self.Site == other.Site && (self.Reachable() || other.Reachable()) {
peers = append(peers, Peer{
Name: other.Name, Key: other.Key,
Endpoint: other.Endpoint,
Allowed: other.Address + "/32",
Keepalive: !self.Reachable(),
Why: "at the same site",
})
}
}
if !self.Hub {
// Everything else goes through the hub, including a node that roams. AllowedIPs is
// the whole overlay, so this is the route of last resort — and because direct peers
// above are single addresses, they win on specificity without either being ambiguous.
why := "the hub — everything not at this site"
if hubIsHere {
// One entry doing both jobs: the direct path to a machine that happens to be
// here, and the route to everywhere else. Splitting them would need two entries
// for one key, which is the thing being avoided.
why = "the hub, which is also at this site — everything goes here"
}
peers = append(peers, Peer{
Name: hub.Name, Key: hub.Key,
Endpoint: hub.Endpoint,
Allowed: overlayCIDR,
Keepalive: !self.Reachable(),
Why: why,
})
} else {
// The hub holds every node that does not share a site with it, because those nodes
// route through it and it must know where to send the replies. Ones it cannot dial
// will dial it.
enrolled := map[string]bool{}
for _, other := range usable {
enrolled[other.Key] = true
if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
continue
}
peers = append(peers, Peer{
Name: other.Name, Key: other.Key,
Endpoint: other.Endpoint,
Allowed: other.Address + "/32",
Why: "routes through this hub",
})
}
// And every peer of the tunnel it took over that has not enrolled (novox/hq ADR
// 0105): the same key and the same address the found tunnel had for it, so the
// machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in,
// as it always did. Once a node enrols with that key, the node's entry above is the
// peer, and WireGuard takes one entry per key.
for _, c := range self.Carried {
if enrolled[c.Key] {
continue
}
peers = append(peers, Peer{
Name: CarriedName(c.Key), Key: c.Key,
Allowed: HostPrefix(c.Address),
Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh",
})
}
}
sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
graph[self.Name] = peers
}
return graph, nil
}
internal/overlay/graph_test.go
-333
View File
@@ -1,333 +0,0 @@
package overlay
import (
"errors"
"strings"
"testing"
)
const cidr = "10.42.0.0/16"
func at(name, site, address, endpoint string, hub bool) Node {
return Node{Name: name, Key: "key-" + name, Site: site, Address: address,
Endpoint: endpoint, Hub: hub}
}
func peersOf(t *testing.T, g Graph, node string) map[string]Peer {
t.Helper()
out := map[string]Peer{}
for _, p := range g[node] {
out[p.Name] = p
}
return out
}
func TestEveryNodeReachesTheHub(t *testing.T) {
// The property that makes this a network at all. Without it a node has no route to anything
// it does not share a site with.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
at("home", "house", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"laptop", "home"} {
hub, ok := peersOf(t, g, node)["anchor"]
if !ok {
t.Fatalf("%s has no route to the hub", node)
}
if hub.Allowed != cidr {
t.Errorf("%s routes %s through the hub; it must be the whole overlay or the hub is "+
"not a route of last resort", node, hub.Allowed)
}
}
}
func TestNodesAtOneSitePeerDirectly(t *testing.T) {
// Machines that share a site have a path that does not need the hub, and using it keeps their
// traffic off a link that may be on another continent.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
at("server", "house", "10.42.0.3", "192.0.2.3:51820", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
direct, ok := peersOf(t, g, "desk")["server"]
if !ok {
t.Fatal("two machines at one site do not peer directly")
}
if direct.Allowed != "10.42.0.3/32" {
t.Errorf("the direct peer allows %s; a single address is what makes it win on "+
"specificity over the hub's whole-overlay route", direct.Allowed)
}
}
func TestARoamingNodeGetsExactlyOnePath(t *testing.T) {
// Hub-only, and not as a simplification. WireGuard has no failover: a more specific route to
// a dead endpoint blackholes rather than falling back, so two paths would mean one of them
// silently swallowing traffic.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
at("other", "", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if got := len(g["laptop"]); got != 1 {
t.Fatalf("a roaming node has %d peers; it gets exactly one path", got)
}
if g["laptop"][0].Name != "anchor" {
t.Errorf("a roaming node's single path is %s, not the hub", g["laptop"][0].Name)
}
}
func TestTwoRoamingNodesDoNotPeerWithEachOther(t *testing.T) {
// An empty site is not a site. Nodes that roam have no shared location, and treating "" as
// one would have every roaming machine try to dial every other, none of which can be dialled.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
at("phone", "", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if _, ok := peersOf(t, g, "laptop")["phone"]; ok {
t.Error("two nodes with no site peered as though they shared one")
}
}
func TestOnlyTheSideThatCannotBeDialledKeepsThePathOpen(t *testing.T) {
// Keepalive matters exactly once: on the node behind NAT. Without it the peer's first packet
// arrives at a mapping that has already expired. On the reachable side it is pointless
// traffic for ever.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if !peersOf(t, g, "laptop")["anchor"].Keepalive {
t.Error("a node that cannot be dialled does not keep its path open")
}
if peersOf(t, g, "anchor")["laptop"].Keepalive {
t.Error("a reachable node sends keepalives it does not need")
}
// And between two direct peers at one site, where whether to keep the path open depends on
// which end you are. Checked here because the hub's own peer list happens not to set the
// field at all, so asserting only against the hub tests an absence rather than the rule.
same, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
at("server", "house", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if !peersOf(t, same, "server")["desk"].Keepalive {
t.Error("the peer that cannot be dialled does not keep the path open")
}
if peersOf(t, same, "desk")["server"].Keepalive {
t.Error("the peer that can be dialled sends keepalives it does not need")
}
}
func TestTheHubKnowsEveryoneItRoutesFor(t *testing.T) {
// The replies have to get back. A hub that does not hold a peer cannot answer it.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
at("home", "house", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
hub := peersOf(t, g, "anchor")
for _, want := range []string{"laptop", "home"} {
if _, ok := hub[want]; !ok {
t.Errorf("the hub does not hold %s, so replies to it have nowhere to go", want)
}
}
}
func TestANodeWithNoPlaceYetIsSkippedRatherThanFatal(t *testing.T) {
// A node that has enrolled and not yet been given an address is an ordinary in-between state.
// Failing the whole graph over it would mean no node gets a network because one is half done.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("laptop", "", "10.42.0.2", "", false),
{Name: "newcomer", Key: "", Address: ""},
}, cidr)
if err != nil {
t.Fatal(err)
}
if _, ok := g["newcomer"]; ok {
t.Error("a node with no key or address was given a peer list")
}
if _, ok := peersOf(t, g, "laptop")["newcomer"]; ok {
t.Error("a node with no key was put in somebody's peer list")
}
}
func TestAHubAtYourOwnSiteAppearsOnceNotTwice(t *testing.T) {
// WireGuard takes one entry per public key. A hub that shares a site with a spoke was being
// emitted twice — once as a direct peer and once as the route of last resort — producing a
// configuration the interface refuses, from a mesh that thought it had succeeded.
//
// Found in the lab on the first two machines that shared a site with their hub, which is the
// ordinary case for a small mesh and was in none of the tests above.
g, err := Compute([]Node{
at("anchor", "lab", "10.42.0.1", "192.0.2.10:51820", true),
at("laptop", "lab", "10.42.0.2", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
seen := map[string]int{}
for _, p := range g["laptop"] {
seen[p.Key]++
}
for key, count := range seen {
if count > 1 {
t.Errorf("the key %s appears %d times; WireGuard takes one entry per key", key, count)
}
}
if len(g["laptop"]) != 1 {
t.Fatalf("laptop has %d peers; the hub at its own site is one peer, not two", len(g["laptop"]))
}
// And that single entry has to carry everything, or the node keeps a direct path to the hub
// and loses its route to everywhere else.
if got := g["laptop"][0].Allowed; got != cidr {
t.Errorf("the single entry allows %s; it is both the direct path and the route of last "+
"resort, so it carries the whole overlay", got)
}
// And nobody peers with themselves, here or in a site of two dialable nodes.
other, err := Compute([]Node{
at("anchor", "house", "10.42.0.1", "198.51.100.1:51820", true),
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
for _, graph := range []Graph{g, other} {
for node, peers := range graph {
for _, p := range peers {
if p.Name == node {
t.Errorf("%s peers with itself", node)
}
}
}
}
}
func TestTwoUnreachableNodesAtOneSiteDoNotPeerDirectly(t *testing.T) {
// Nobody would open the path, and the direct route is more specific than the hub's — so it
// wins and blackholes. Both nodes would reach the hub perfectly and be unable to reach each
// other, which is the worst arrangement available: it looks configured and is not.
//
// This is the design's own warning arriving in its implementation, and the lab found it with
// two machines at one site behind no reachable address — the ordinary shape of a house.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("desk", "house", "10.42.0.2", "", false),
at("server", "house", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if _, ok := peersOf(t, g, "desk")["server"]; ok {
t.Error("two nodes that neither can be dialled were peered directly; neither could open " +
"the path and the route is more specific than the hub's, so it blackholes")
}
// And they must still be able to reach each other — through the hub.
if hub := peersOf(t, g, "desk")["anchor"]; hub.Allowed != cidr {
t.Errorf("desk's route to everything else allows %s", hub.Allowed)
}
}
func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) {
// The other side of it: if either end can be dialled, the path can be opened, and using it
// keeps their traffic off a hub that may be on another continent.
g, err := Compute([]Node{
at("anchor", "datacentre", "10.42.0.1", "198.51.100.1:51820", true),
at("desk", "house", "10.42.0.2", "192.0.2.2:51820", false),
at("server", "house", "10.42.0.3", "", false),
}, cidr)
if err != nil {
t.Fatal(err)
}
if _, ok := peersOf(t, g, "server")["desk"]; !ok {
t.Error("a node did not peer with a reachable neighbour at its own site")
}
if _, ok := peersOf(t, g, "desk")["server"]; !ok {
t.Error("the reachable node did not hold its unreachable neighbour, so replies have " +
"nowhere to go")
}
}
// novox/hq ADR 0105: a hub that took over the predecessor's tunnel carries every peer that tunnel
// had, under the key and at the address the peer knows, until a node enrols with that key.
func TestTheHubCarriesTheTunnelsPeersUntilTheyEnrol(t *testing.T) {
hub := at("anchor", "hosting", "192.0.2.1", "198.51.100.1:51900", true)
hub.Carried = []Carried{
{Key: "key-home", Address: "192.0.2.2"},
{Key: "key-workstation", Address: "192.0.2.3"},
}
// The machine behind key-home enrolled: it is a node now, at the address it kept.
home := at("home", "house", "192.0.2.2", "", false)
home.Key = "key-home"
g, err := Compute([]Node{hub, home}, "192.0.2.0/24")
if err != nil {
t.Fatal(err)
}
peers := peersOf(t, g, "anchor")
carried, ok := peers[CarriedName("key-workstation")]
if !ok {
t.Fatalf("the hub does not carry the peer that has not enrolled: %+v", g["anchor"])
}
if carried.Allowed != "192.0.2.3/32" || carried.Endpoint != "" || carried.Key != "key-workstation" {
t.Errorf("a carried peer is not the tunnel's own entry — same key, its one address, no endpoint: %+v", carried)
}
if _, twice := peers[CarriedName("key-home")]; twice {
t.Error("a peer that enrolled is carried as well as listed as a node: WireGuard takes one entry per key")
}
if node, ok := peers["home"]; !ok || node.Key != "key-home" || node.Allowed != "192.0.2.2/32" {
t.Errorf("the enrolled peer is not the node it became: %+v", node)
}
// Carried peers are the hub's business only: a spoke routes everything through the hub.
if _, leaked := peersOf(t, g, "home")[CarriedName("key-workstation")]; leaked {
t.Error("a carried peer appeared in a spoke's peer list")
}
}
// A mesh whose hub cannot carry the paths between sites is refused, never answered as an empty graph: with no hub there
// is no path between sites, and a hub with no endpoint is a peer every node is given and none can reach. Either would
// look configured and be silent, which is how the old arrangement failed when nobody knew the address convention.
func TestAHubThatCannotCarryThePathsIsRefused(t *testing.T) {
for _, c := range []struct {
name string
nodes []Node
says func(error) bool
}{
{"no hub", []Node{at("a", "", "10.42.0.1", "", false), at("b", "", "10.42.0.2", "", false)},
func(err error) bool { return errors.Is(err, ErrNoHub) }},
{"a hub with no endpoint", []Node{at("anchor", "datacentre", "10.42.0.1", "", true), at("laptop", "", "10.42.0.2", "", false)},
func(err error) bool { return err != nil && strings.Contains(err.Error(), "must be reachable") }},
} {
if _, err := Compute(c.nodes, cidr); !c.says(err) {
t.Errorf("%s: %v", c.name, err)
}
}
}
internal/overlay/names.go
-30
View File
@@ -1,30 +0,0 @@
package overlay
import (
sdk "git.novox.be/novox/mesh-sdk/go/overlay"
)
// The private network's names and the shape of its module stay in mesh-sdk's overlay package, which more parts read
// (novox/hq ADR 0593 §1); the generator of each node's network configuration is the node-engine's own, here. These
// are those names, so the generator and the composition read one package.
const (
Requirement = sdk.Requirement
Name = sdk.Name
Addressing = sdk.Addressing
TheNetwork = sdk.TheNetwork
SuffixVar = sdk.SuffixVar
DefaultSuffix = sdk.DefaultSuffix
Interface = sdk.Interface
ConfigPath = sdk.ConfigPath
Unit = sdk.Unit
DefaultKeyPath = sdk.DefaultKeyPath
)
// Manifest is the module the mesh provides for itself (sdk.Manifest).
func Manifest() map[string]any { return sdk.Manifest() }
// Suffix is the internal names' suffix (sdk.Suffix).
func Suffix() string { return sdk.Suffix() }
// InternalName is a node's internal name (sdk.InternalName).
func InternalName(node string) string { return sdk.InternalName(node) }
internal/overlay/opens_test.go
-113
View File
@@ -1,113 +0,0 @@
package overlay
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
func networkOf(t *testing.T, nodes []Node) *Generator {
t.Helper()
made, err := From(nodes, "10.42.0.0/16", "/var/lib/mesh-host/overlay.key")
if err != nil {
t.Fatal(err)
}
return made
}
// A hub is dialled by every node at other sites, and needs its port open. A machine that is not a
// hub dials out and needs nothing open at all.
//
// They are the same module, which is why a static field in a manifest cannot say it — and the
// machine it gets wrong is the one facing the public internet, which is the machine that most
// needs filtering.
func TestOnlyAMachineThatCanBeDialledOpensTheOverlaysPort(t *testing.T) {
network := networkOf(t, []Node{
{Name: "anchor", Key: "a", Endpoint: "198.51.100.10:51820", Site: "one", Hub: true},
{Name: "laptop", Key: "b", Site: "one"},
})
opens, err := network.Listens("anchor")
if err != nil {
t.Fatal(err)
}
if len(opens) != 1 {
t.Fatalf("the machine every other one dials opens %d ports", len(opens))
}
if opens[0].Port != 51820 || opens[0].At() != "udp" {
t.Fatalf("the port is not the one the interface listens on: %+v", opens[0])
}
// From everywhere, and deliberately: a node at another site is not on the private network
// until this port lets it on, so restricting it to the mesh would be a rule that can never be
// satisfied by the thing it exists for.
if opens[0].From != catalogue.FromEverywhere {
t.Fatalf("the way onto the private network is restricted to the private network: %+v", opens[0])
}
if opens[0].Why == "" {
t.Fatal("a port is opened and nothing says why, which is what makes a rule set unreadable")
}
// And the machine nothing dials opens nothing. Not harmless to get wrong: a rule with no
// reason is one somebody later has to work out the reason for.
quiet, err := network.Listens("laptop")
if err != nil {
t.Fatal(err)
}
if len(quiet) != 0 {
t.Fatalf("a machine nothing dials opened %d port(s)", len(quiet))
}
}
// The port comes from the endpoint, which is where the interface takes its ListenPort from. One
// source, so a rule set cannot open a port the interface is not on.
func TestTheOpenedPortIsTheOneTheInterfaceListensOn(t *testing.T) {
network := networkOf(t, []Node{
{Name: "anchor", Key: "a", Endpoint: "198.51.100.10:60000", Site: "one", Hub: true},
})
opens, err := network.Listens("anchor")
if err != nil {
t.Fatal(err)
}
if len(opens) != 1 || opens[0].Port != 60000 {
t.Fatalf("the rule set would open a port the interface is not on: %+v", opens)
}
written, err := Declaration(Node{Name: "anchor", Key: "a", Address: "10.42.0.1",
Endpoint: "198.51.100.10:60000", Hub: true}, nil, "/k")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(written), "ListenPort = 60000") {
t.Fatalf("the interface and the rule set disagree about the port:\n%s", written)
}
}
// An endpoint with no port is refused rather than treated as a machine that opens nothing.
//
// A generator that cannot say what a machine opens is not one that says it opens nothing, and
// closing a port on the evidence of a failure to look is how a machine is severed by a fault
// somewhere else entirely.
func TestAnEndpointWithNoPortIsRefusedRatherThanTakenAsSilence(t *testing.T) {
network := networkOf(t, []Node{
{Name: "anchor", Key: "a", Endpoint: "198.51.100.10", Site: "one", Hub: true},
})
if _, err := network.Listens("anchor"); err == nil {
t.Fatal("a machine whose port could not be read was treated as opening nothing")
}
}
// A machine the network has never heard of opens nothing, and that is an answer rather than an
// error: a node assigned the module before it is placed is in exactly that state.
func TestAMachineNotOnTheNetworkOpensNothing(t *testing.T) {
network := networkOf(t, []Node{
{Name: "anchor", Key: "a", Endpoint: "198.51.100.10:51820", Site: "one", Hub: true},
})
opens, err := network.Listens("a-stranger")
if err != nil {
t.Fatal(err)
}
if len(opens) != 0 {
t.Fatalf("a machine not on the network opened %d port(s)", len(opens))
}
}